Fixed e-mail reading order, compatibility to exchange 2010 and recursively add attachments

requirements.txt

@@ -1,5 +1,6 @@
 -i https://pypi.org/simple
 asn1crypto==0.24.0
+bs4==0.0.1
 cached-property==1.5.1
 certifi==2018.8.24
 cffi==1.11.5

workflows/Ews2Case.py

@@ -11,6 +11,7 @@
 from objects.EwsConnector import EwsConnector
 from objects.TheHiveConnector import TheHiveConnector
 from objects.TempAttachment import TempAttachment
+from bs4 import BeautifulSoup
 
 def connectEws():
     logger = logging.getLogger(__name__)
@@ -28,7 +29,7 @@ def connectEws():
 
         theHiveConnector = TheHiveConnector(cfg)
 
-        for msg in unread:
+        for msg in reversed(unread):
             #type(msg)
             #<class 'exchangelib.folders.Message'>
             conversationId = msg.conversation_id.id
@@ -75,44 +76,10 @@ def connectEws():
             fullBody = getEmailBody(msg)
             taskLog = theHiveConnector.craftTaskLog(fullBody)
             createdTaskLogId = theHiveConnector.addTaskLog(commTaskId, taskLog)
-
+            attachedFiles = getFileAttachments(msg)
+            for attached in attachedFiles:
+                theHiveConnector.addFileObservable(esCaseId, attached['data'], attached['message'])
             readMsg = ewsConnector.markAsRead(msg)
-
-            for attachmentLvl1 in msg.attachments:
-                #uploading the attachment as file observable
-                #is the attachment is a .msg, the eml version
-                #of the file is uploaded
-                tempAttachment = TempAttachment(attachmentLvl1)
-
-                if not tempAttachment.isInline:
-                    #adding the attachment only if it is not inline
-                    #inline attachments are pictures in the email body
-                    tmpFilepath = tempAttachment.writeFile()
-                    to = str()
-                    for recipient in msg.to_recipients:
-                        to = to + recipient.email_address + ' ' 
-                    comment = 'Attachment from email sent by '
-                    comment += str(msg.author.email_address).lower()
-                    comment += ' and received by '
-                    comment += str(to).lower()
-                    comment += ' with subject: <'
-                    comment += msg.subject
-                    comment += '>'
-                    theHiveConnector.addFileObservable(esCaseId,
-                        tmpFilepath,
-                        comment)
-
-                    if tempAttachment.isEmailAttachment:
-                        #if the attachment is an email
-                        #attachments of this email are also
-                        #uploaded to TheHive
-                        for attachmentLvl2 in tempAttachment.attachments:
-                            tempAttachmentLvl2 = TempAttachment(attachmentLvl2)
-                            tmpFilepath = tempAttachmentLvl2.writeFile()
-                            comment = 'Attachment from the email attached'
-                            theHiveConnector.addFileObservable(esCaseId,
-                                tmpFilepath,
-                                comment)
 
         report['success'] = True
         return report
@@ -122,6 +89,36 @@ def connectEws():
             report['success'] = False
             return report
 
+def getFileAttachments(msg):
+    files = []
+    for attachmentLvl1 in msg.attachments:
+        #uploading the attachment as file observable
+        #if the attachment is a .msg, the eml version of the file is uploaded
+        tempAttachment = TempAttachment(attachmentLvl1)
+
+        if not tempAttachment.isInline:
+            #adding the attachment only if it is not inline
+            #inline attachments are pictures in the email body
+            tmpFilepath = tempAttachment.writeFile()
+
+            to = str()
+            for recipient in msg.to_recipients:
+                to = to + recipient.email_address + ' ' 
+            comment = 'Attachment from email sent by '
+            comment += str(msg.author.email_address).lower()
+            comment += ' and received by '
+            comment += str(to).lower()
+            comment += ' with subject: <'
+            comment += msg.subject
+            comment += '>'
+            files.append({
+                    "data":tmpFilepath,
+                    "message":comment
+                })
+            if tempAttachment.isEmailAttachment:
+                #recursively extracts attachments from attached emails
+                files.extend(getFileAttachments(attachmentLvl1.item))
+    return files
 
 def getEmailBody(email):
     #crafting some "reply to" info
@@ -136,7 +133,7 @@ def getEmailBody(email):
     #because cannot iterate over None object
     if email.to_recipients:
         for recipient in email.to_recipients:
-            to = to + recipient.email_address + ' ' 
+            to = to + recipient.email_address + ' '
     else:
         to = ''
 
@@ -148,14 +145,16 @@ def getEmailBody(email):
 
     body = email.text_body
 
-    #alternate way to get the body
-    #soup = BeautifulSoup(email.body, 'html.parser')
-    #try:
-    #    #html email
-    #    body = soup.body.text
-    #except AttributeError:
-    #    #non html email
-    #    body = soup.text
+    #exchange 2010 doesn't have attribute text_body, need to treat body as html
+    if body is None:
+        #alternate way to get the body
+        soup = BeautifulSoup(msg.body, 'html.parser')
+        try:
+            #html email
+            body = soup.body.text
+        except AttributeError:
+            #non html email
+            body = soup.text
 
     return ('```\n' + replyToInfo + body + '\n```')
 

workflows/objects/TheHiveConnector.py

@@ -166,6 +166,9 @@ def addFileObservable(self, esCaseId, filepath, comment):
         if response.status_code == 201:
             esObservableId = response.json()['id']
             return esObservableId
+        # ignores the attachment if is already in the case
+        elif response.status_code == 400 and response.json().get('message', '') == 'Artifact already exists':
+            return None
         else:
             self.logger.error('File observable upload failed')
             raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True))