[MRG] Merge pull request #528 from dfir-iris/architecture_improvements

Architecture improvements

CODESTYLE.md

@@ -61,6 +61,7 @@ New files should be prefixed by the following license header, where `${current_y
   This allows any code maintainer to immediately spot which code elements can be freely modified
   without having to worry about the external context.
   Note: private elements are only called within the modules in which they are defined.
+* Function names should be prefixed by the module name they belong to. Example: `iocs_create` instead of `create`
 
 ## Javascript coding rules
 

architecture.md

@@ -0,0 +1,55 @@
+# Iris Architecture
+
+The IRIS coarse-grained architecture can be understood by looking at the docker-compose.yml file. The main elements are:
+
+* db: postgresql database to store all application data
+* app: backend application
+* worker: most module hooks are processed by the worker 
+* rabbitmq: message broker between the app and worker
+* nginx: the front server to serve static files and dispatch requests to app
+
+## Code organisation
+
+This section explains how the code is organized within the major namespaces.
+They reflect the layered architecture of the IRIS backend:
+
+* blueprints
+* business
+* datamgmt
+
+The IRIS backend is a Flask application.
+
+### blueprints
+
+This is the public API of the `app`. It contains all the endpoints: REST, GraphQL, Flask templates (pages and modals). 
+The requests payloads are converted to business objects from `models` and passed down to calls into the business layer.
+
+Forbidden imports in this layer:
+
+* `from app.datamgmt`, as everything should go through the business layer first 
+* `from sqlalchemy`
+
+### business
+
+This is where processing happens. The methods should exclusively manipulate business objects from the `models` namespace.
+
+Forbidden imports in this layer:
+
+* `from app import db`, as the business layer should not take case of persistence details but rather delegate to the
+  `datamgmt` layer
+
+### datamgmt
+
+This layer handles persistence. It should be the only layer with knowledge of the database engine.
+
+Forbidden imports in this layer:
+
+* `from app.business`, as the business layer should call the persistence layer (not the other way around)
+
+### models
+
+The description of all objects handled by IRIS `business` layer and persisted through `datamgt`.
+
+### alembic
+
+This namespace takes care of the database migration. 

docker-compose.dev.yml

@@ -42,6 +42,8 @@ services:
     image: iriswebapp_app:v2.4.7
     ports:
       - "127.0.0.1:8000:8000"
+    volumes:
+      - ./source/app:/iriswebapp/app
 
   worker:
     extends:
@@ -51,6 +53,8 @@ services:
       context: .
       dockerfile: docker/webApp/Dockerfile
     image: iriswebapp_app:v2.4.7
+    volumes:
+      - ./source/app:/iriswebapp/app
 
   nginx:
     extends:

source/app/blueprints/case/case_ioc_routes.py

@@ -16,7 +16,6 @@
 #  along with this program; if not, write to the Free Software Foundation,
 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-# IMPORTS ------------------------------------------------
 from datetime import datetime
 
 import csv
@@ -61,9 +60,9 @@
 from app.util import ac_case_requires
 from app.util import response_error
 from app.util import response_success
-from app.business.iocs import create
-from app.business.iocs import update
-from app.business.iocs import delete
+from app.business.iocs import iocs_create
+from app.business.iocs import iocs_update
+from app.business.iocs import iocs_delete
 from app.business.errors import BusinessProcessingError
 
 case_ioc_blueprint = Blueprint(
@@ -73,7 +72,6 @@
 )
 
 
-# CONTENT ------------------------------------------------
 @case_ioc_blueprint.route('/case/ioc', methods=['GET'])
 @ac_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access)
 def case_ioc(caseid, url_redir):
@@ -130,7 +128,7 @@ def case_add_ioc(caseid):
     ioc_schema = IocSchema()
 
     try:
-        ioc, msg = create(request.get_json(), caseid)
+        ioc, msg = iocs_create(request.get_json(), caseid)
         return response_success(msg, data=ioc_schema.dump(ioc))
     except BusinessProcessingError as e:
         return response_error(e.get_message(), data=e.get_data())
@@ -249,7 +247,7 @@ def case_add_ioc_modal(caseid):
 def case_delete_ioc(cur_id, caseid):
     try:
 
-        msg = delete(cur_id, caseid)
+        msg = iocs_delete(cur_id, caseid)
         return response_success(msg=msg)
 
     except BusinessProcessingError as e:
@@ -297,7 +295,7 @@ def case_update_ioc(cur_id, caseid):
     ioc_schema = IocSchema()
 
     try:
-        ioc, msg = update(cur_id, request.get_json(), caseid)
+        ioc, msg = iocs_update(cur_id, request.get_json(), caseid)
         return response_success(msg, data=ioc_schema.dump(ioc))
     except BusinessProcessingError as e:
         return response_error(e.get_message(), data=e.get_data())

source/app/blueprints/case/case_notes_routes.py

@@ -17,9 +17,8 @@
 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import marshmallow
-# IMPORTS ------------------------------------------------
 from datetime import datetime
-from flask import Blueprint, jsonify
+from flask import Blueprint
 from flask import redirect
 from flask import render_template
 from flask import request
@@ -31,12 +30,18 @@
 
 from app import db, socket_io, app
 from app.blueprints.case.case_comments import case_comment_update
-from app.business.errors import BusinessProcessingError, UnhandledBusinessError
-from app.business.notes import update, create, list_note_revisions, get_note_revision, delete_note_revision
+from app.business.errors import BusinessProcessingError
+from app.business.notes import notes_update
+from app.business.notes import notes_create
+from app.business.notes import notes_list_revisions
+from app.business.notes import notes_get_revision
+from app.business.notes import notes_delete_revision
 from app.datamgmt.case.case_db import case_get_desc_crc
 from app.datamgmt.case.case_db import get_case
-from app.datamgmt.case.case_notes_db import add_comment_to_note, get_directories_with_note_count, get_directory, \
-    delete_directory
+from app.datamgmt.case.case_notes_db import add_comment_to_note
+from app.datamgmt.case.case_notes_db import get_directories_with_note_count
+from app.datamgmt.case.case_notes_db import get_directory
+from app.datamgmt.case.case_notes_db import delete_directory
 from app.datamgmt.case.case_notes_db import delete_note
 from app.datamgmt.case.case_notes_db import delete_note_comment
 from app.datamgmt.case.case_notes_db import get_case_note_comment
@@ -47,10 +52,13 @@
 from app.iris_engine.utils.tracker import track_activity
 from app.models import Notes
 from app.models.authorization import CaseAccessLevel
-from app.schema.marshables import CaseNoteDirectorySchema, CaseNoteRevisionSchema
+from app.schema.marshables import CaseNoteDirectorySchema
+from app.schema.marshables import CaseNoteRevisionSchema
 from app.schema.marshables import CaseNoteSchema
 from app.schema.marshables import CommentSchema
-from app.util import ac_api_case_requires, ac_socket_requires, endpoint_deprecated, add_obj_history_entry
+from app.util import ac_api_case_requires
+from app.util import ac_socket_requires
+from app.util import endpoint_deprecated
 from app.util import ac_case_requires
 from app.util import response_error
 from app.util import response_success
@@ -61,7 +69,6 @@
                                  template_folder='templates')
 
 
-# CONTENT ------------------------------------------------
 @case_notes_blueprint.route('/case/notes', methods=['GET'])
 @ac_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access)
 def case_notes(caseid, url_redir):
@@ -158,9 +165,7 @@ def case_note_save(cur_id, caseid):
 
     try:
 
-        note = update(identifier=cur_id,
-                      request_json=request.get_json(),
-                      case_identifier=caseid)
+        note = notes_update(identifier=cur_id, request_json=request.get_json(), case_identifier=caseid)
 
         return response_success(f"Note ID {cur_id} saved", data=addnote_schema.dump(note))
 
@@ -175,8 +180,7 @@ def case_note_list_history(cur_id, caseid):
 
     try:
 
-        note_version = list_note_revisions(identifier=cur_id,
-                                           case_identifier=caseid)
+        note_version = notes_list_revisions(identifier=cur_id, case_identifier=caseid)
 
         return response_success(f"ok", data=note_version_sc.dump(note_version))
 
@@ -191,9 +195,9 @@ def case_note_revision(cur_id, revision_id, caseid):
 
     try:
 
-        note_version = get_note_revision(identifier=cur_id,
-                                         revision_number=revision_id,
-                                         case_identifier=caseid)
+        note_version = notes_get_revision(identifier=cur_id,
+                                          revision_number=revision_id,
+                                          case_identifier=caseid)
 
         return response_success(f"ok", data=note_version_sc.dump(note_version))
 
@@ -207,9 +211,9 @@ def case_note_revision_delete(cur_id, revision_id, caseid):
 
     try:
 
-        delete_note_revision(identifier=cur_id,
-                             revision_number=revision_id,
-                             case_identifier=caseid)
+        notes_delete_revision(identifier=cur_id,
+                              revision_number=revision_id,
+                              case_identifier=caseid)
 
         return response_success(f"Revision {revision_id} of note {cur_id} deleted")
 
@@ -224,8 +228,7 @@ def case_note_add(caseid):
 
     try:
 
-        note = create(request_json=request.get_json(),
-                      case_identifier=caseid)
+        note = notes_create(request_json=request.get_json(), case_identifier=caseid)
 
         return response_success(f"Note ID {note.note_id} created", data=addnote_schema.dump(note))
 
@@ -525,7 +528,7 @@ def socket_ping_note(data):
 
 @socket_io.on('pong-note')
 @ac_socket_requires(CaseAccessLevel.full_access)
-def socket_ping_note(data):
+def socket_pong_note(data):
 
     emit('pong-note', {"user": current_user.name, "note_id": data['note_id']}, room=data['channel'])
 

source/app/blueprints/case/case_routes.py

@@ -18,7 +18,6 @@
 
 import binascii
 import marshmallow
-# IMPORTS ------------------------------------------------
 import traceback
 from flask import Blueprint
 from flask import redirect
@@ -54,19 +53,23 @@
 from app.datamgmt.manage.manage_users_db import get_user
 from app.datamgmt.manage.manage_users_db import get_users_list_restricted_from_case
 from app.datamgmt.manage.manage_users_db import set_user_case_access
-from app.datamgmt.reporter.report_db import export_case_json
+from app.business.cases import cases_export_to_json
 from app.forms import PipelinesCaseForm
-from app.iris_engine.access_control.utils import ac_get_all_access_level, ac_fast_check_current_user_has_case_access, \
-    ac_fast_check_user_has_case_access
+from app.iris_engine.access_control.utils import ac_get_all_access_level
+from app.iris_engine.access_control.utils import ac_fast_check_user_has_case_access
 from app.iris_engine.access_control.utils import ac_set_case_access_for_users
 from app.iris_engine.module_handler.module_handler import list_available_pipelines
 from app.iris_engine.utils.tracker import track_activity
-from app.models import CaseStatus, ReviewStatusList
+from app.models import CaseStatus
+from app.models import ReviewStatusList
 from app.models import UserActivity
 from app.models.authorization import CaseAccessLevel
 from app.models.authorization import User
-from app.schema.marshables import TaskLogSchema, CaseSchema, CaseDetailsSchema
-from app.util import ac_api_case_requires, add_obj_history_entry
+from app.schema.marshables import TaskLogSchema
+from app.schema.marshables import CaseSchema
+from app.schema.marshables import CaseDetailsSchema
+from app.util import ac_api_case_requires
+from app.util import add_obj_history_entry
 from app.util import ac_case_requires
 from app.util import ac_socket_requires
 from app.util import response_error
@@ -168,7 +171,7 @@ def socket_summary_onsave(data):
 
 @socket_io.on('clear_buffer')
 @ac_socket_requires(CaseAccessLevel.full_access)
-def socket_summary_onchange(message):
+def socket_summary_on_clear_buffer(message):
 
     emit('clear_buffer', message)
 
@@ -240,7 +243,7 @@ def activity_fetch(caseid):
 @case_blueprint.route("/case/export", methods=['GET'])
 @ac_api_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access)
 def export_case(caseid):
-    return response_success('', data=export_case_json(caseid))
+    return response_success('', data=cases_export_to_json(caseid))
 
 
 @case_blueprint.route("/case/meta", methods=['GET'])

source/app/blueprints/graphql/cases.py

@@ -27,11 +27,16 @@
 from graphene import Float
 from graphene import String
 
-from app.business.iocs import build_filter_case_ioc_query
 from app.models.cases import Cases
-from app.business.cases import create
-from app.business.cases import delete
-from app.business.cases import update
+from app.models.authorization import Permissions
+from app.models.authorization import CaseAccessLevel
+
+from app.business.iocs import iocs_build_filter_query
+from app.business.cases import cases_create
+from app.business.cases import cases_delete
+from app.business.cases import cases_update
+from app.business.permissions import permissions_check_current_user_has_some_permission
+from app.business.permissions import permissions_check_current_user_has_some_case_access
 
 from app.blueprints.graphql.iocs import IOCConnection
 
@@ -48,10 +53,10 @@ class Meta:
     @staticmethod
     def resolve_iocs(root, info, ioc_id=None, ioc_uuid=None, ioc_value=None, ioc_type_id=None, ioc_description=None, ioc_tlp_id=None, ioc_tags=None,
                      ioc_misp=None, user_id=None, Linked_cases=None, **kwargs):
-        return build_filter_case_ioc_query(ioc_id=ioc_id, ioc_uuid=ioc_uuid, ioc_value=ioc_value,
-                                           ioc_type_id=ioc_type_id, ioc_description=ioc_description,
-                                           ioc_tlp_id=ioc_tlp_id, ioc_tags=ioc_tags, ioc_misp=ioc_misp,
-                                           user_id=user_id, linked_cases=Linked_cases)
+        return iocs_build_filter_query(ioc_id=ioc_id, ioc_uuid=ioc_uuid, ioc_value=ioc_value,
+                                       ioc_type_id=ioc_type_id, ioc_description=ioc_description,
+                                       ioc_tlp_id=ioc_tlp_id, ioc_tags=ioc_tags, ioc_misp=ioc_misp,
+                                       user_id=user_id, linked_cases=Linked_cases)
 
 
 class CaseConnection(Connection):
@@ -89,7 +94,7 @@ def mutate(root, info, name, description, client_id, soc_id=None, classification
             request['case_soc_id'] = soc_id
         if classification_id:
             request['classification_id'] = classification_id
-        case, _ = create(request)
+        case, _ = cases_create(request)
         return CaseCreate(case=case)
 
 
@@ -102,7 +107,9 @@ class Arguments:
 
     @staticmethod
     def mutate(root, info, case_id):
-        delete(case_id)
+        permissions_check_current_user_has_some_permission([Permissions.standard_user])
+        permissions_check_current_user_has_some_case_access(case_id, [CaseAccessLevel.full_access])
+        cases_delete(case_id)
 
 
 class CaseUpdate(Mutation):
@@ -150,5 +157,8 @@ def mutate(root, info, case_id, name=None, soc_id=None, classification_id=None,
             request['case_tags'] = tags
         if review_status_id:
             request['review_status_id'] = review_status_id
-        case, _ = update(case_id, request)
+        permissions_check_current_user_has_some_permission([Permissions.standard_user])
+        permissions_check_current_user_has_some_case_access(case_id, [CaseAccessLevel.full_access])
+
+        case, _ = cases_update(case_id, request)
         return CaseUpdate(case=case)