From 8437b18941c5df254df29b404489f5b5b56ac87d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 18 Apr 2019 15:22:58 -0300 Subject: [PATCH 1/2] Reads e-mails from the oldest to newest Using Beautifulsoap to read exchange 2010 and older e-mails (they don't have text_body attribute) --- requirements.txt | 1 + workflows/Ews2Case.py | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 885928c..4737197 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -i https://pypi.org/simple asn1crypto==0.24.0 +bs4==0.0.1 cached-property==1.5.1 certifi==2018.8.24 cffi==1.11.5 diff --git a/workflows/Ews2Case.py b/workflows/Ews2Case.py index d4aced3..9e7e364 100644 --- a/workflows/Ews2Case.py +++ b/workflows/Ews2Case.py @@ -11,6 +11,7 @@ from objects.EwsConnector import EwsConnector from objects.TheHiveConnector import TheHiveConnector from objects.TempAttachment import TempAttachment +from bs4 import BeautifulSoup def connectEws(): logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ def connectEws(): theHiveConnector = TheHiveConnector(cfg) - for msg in unread: + for msg in reversed(unread): #type(msg) # conversationId = msg.conversation_id.id @@ -148,14 +149,16 @@ def getEmailBody(email): body = email.text_body - #alternate way to get the body - #soup = BeautifulSoup(email.body, 'html.parser') - #try: - # #html email - # body = soup.body.text - #except AttributeError: - # #non html email - # body = soup.text + #exchange 2010 doesn't have attribute text_body, need to treat body as html + if body is None: + #alternate way to get the body + soup = BeautifulSoup(msg.body, 'html.parser') + try: + #html email + body = soup.body.text + except AttributeError: + #non html email + body = soup.text return ('```\n' + replyToInfo + body + '\n```') From e66661e7d56464b69d0103cdf2fc2664a42ab827 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 29 May 2019 18:48:14 -0300 Subject: [PATCH 2/2] Includes attachments now recursively, not only first level Ignores if the artifact is already in the case, instead of getting an error --- workflows/Ews2Case.py | 72 +++++++++++++-------------- workflows/objects/TheHiveConnector.py | 3 ++ 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/workflows/Ews2Case.py b/workflows/Ews2Case.py index 9e7e364..8372637 100644 --- a/workflows/Ews2Case.py +++ b/workflows/Ews2Case.py @@ -76,44 +76,10 @@ def connectEws(): fullBody = getEmailBody(msg) taskLog = theHiveConnector.craftTaskLog(fullBody) createdTaskLogId = theHiveConnector.addTaskLog(commTaskId, taskLog) - + attachedFiles = getFileAttachments(msg) + for attached in attachedFiles: + theHiveConnector.addFileObservable(esCaseId, attached['data'], attached['message']) readMsg = ewsConnector.markAsRead(msg) - - for attachmentLvl1 in msg.attachments: - #uploading the attachment as file observable - #is the attachment is a .msg, the eml version - #of the file is uploaded - tempAttachment = TempAttachment(attachmentLvl1) - - if not tempAttachment.isInline: - #adding the attachment only if it is not inline - #inline attachments are pictures in the email body - tmpFilepath = tempAttachment.writeFile() - to = str() - for recipient in msg.to_recipients: - to = to + recipient.email_address + ' ' - comment = 'Attachment from email sent by ' - comment += str(msg.author.email_address).lower() - comment += ' and received by ' - comment += str(to).lower() - comment += ' with subject: <' - comment += msg.subject - comment += '>' - theHiveConnector.addFileObservable(esCaseId, - tmpFilepath, - comment) - - if tempAttachment.isEmailAttachment: - #if the attachment is an email - #attachments of this email are also - #uploaded to TheHive - for attachmentLvl2 in tempAttachment.attachments: - tempAttachmentLvl2 = TempAttachment(attachmentLvl2) - tmpFilepath = tempAttachmentLvl2.writeFile() - comment = 'Attachment from the email attached' - theHiveConnector.addFileObservable(esCaseId, - tmpFilepath, - comment) report['success'] = True return report @@ -123,6 +89,36 @@ def connectEws(): report['success'] = False return report +def getFileAttachments(msg): + files = [] + for attachmentLvl1 in msg.attachments: + #uploading the attachment as file observable + #if the attachment is a .msg, the eml version of the file is uploaded + tempAttachment = TempAttachment(attachmentLvl1) + + if not tempAttachment.isInline: + #adding the attachment only if it is not inline + #inline attachments are pictures in the email body + tmpFilepath = tempAttachment.writeFile() + + to = str() + for recipient in msg.to_recipients: + to = to + recipient.email_address + ' ' + comment = 'Attachment from email sent by ' + comment += str(msg.author.email_address).lower() + comment += ' and received by ' + comment += str(to).lower() + comment += ' with subject: <' + comment += msg.subject + comment += '>' + files.append({ + "data":tmpFilepath, + "message":comment + }) + if tempAttachment.isEmailAttachment: + #recursively extracts attachments from attached emails + files.extend(getFileAttachments(attachmentLvl1.item)) + return files def getEmailBody(email): #crafting some "reply to" info @@ -137,7 +133,7 @@ def getEmailBody(email): #because cannot iterate over None object if email.to_recipients: for recipient in email.to_recipients: - to = to + recipient.email_address + ' ' + to = to + recipient.email_address + ' ' else: to = '' diff --git a/workflows/objects/TheHiveConnector.py b/workflows/objects/TheHiveConnector.py index 630be09..d19e17d 100644 --- a/workflows/objects/TheHiveConnector.py +++ b/workflows/objects/TheHiveConnector.py @@ -166,6 +166,9 @@ def addFileObservable(self, esCaseId, filepath, comment): if response.status_code == 201: esObservableId = response.json()['id'] return esObservableId + # ignores the attachment if is already in the case + elif response.status_code == 400 and response.json().get('message', '') == 'Artifact already exists': + return None else: self.logger.error('File observable upload failed') raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True))