Store the original email in the emails table.

cyrusinnovation · Jun 25, 2014 · c91d0cb · c91d0cb
1 parent 79ac620
commit c91d0cb
Show file tree

Hide file tree

Showing 15 changed files with 41 additions and 30 deletions.
diff --git a/App/Program.fs b/App/Program.fs
@@ -35,7 +35,7 @@ let loadDataFrom (filename: string) =
     let inputString = System.IO.File.ReadAllText(filename)
     let message = loadMimeMessageFrom(inputString)
     let parseFunction = selectParseFunction filename
-    let parsed = parseFunction message
+    let parsed = parseFunction message inputString
     loadMail parsed
     ()
 

diff --git a/DenbowParser/Parser.fs b/DenbowParser/Parser.fs
@@ -131,14 +131,14 @@ let rec calendarEntriesFrom (messageParts: list<MessagePart>) : list<CalendarEnt
         | RsvpLinkPart(_) :: items -> calendarEntry :: (calendarEntriesFrom remainingParts)
         | _ -> sprintf "Nonempty list not starting with RSVP link part: [%s]" (remainingParts.ToString()) |> failwith
 
-let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (messageParts: list<MessagePart>) : EmailData =
+let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (originalMessageString: string) (messageParts: list<MessagePart>) : EmailData =
     let intro = messageParts |> extractWithEmptyStringDefault (function | IntroPart(intro) -> Some(String.concat "\n" intro)  | _ -> None)
     let nonIntroParts = messageParts |> List.filter (function |IntroPart(_) -> false | _ -> true)
     let calendarEntries = calendarEntriesFrom nonIntroParts
 
-    { MailDate = sentDate; MailSender = sender; MailIntro = intro; CalendarEntries = calendarEntries }
+    { MailDate = sentDate; MailSender = sender; MailIntro = intro; OriginalMessage = originalMessageString; CalendarEntries = calendarEntries }
 
-let parseMail (message: MimeMessage) : EmailData = 
-    let messageData = messageDataFor message
+let parseMail (message: MimeMessage) (originalMessageString: string) : EmailData = 
+    let messageData = messageDataFor message originalMessageString
     let messageParts = parse messageData.MessageLines PreIntro
-    parseIntoEmailData messageData.Sender messageData.SentDate messageParts
+    parseIntoEmailData messageData.Sender messageData.SentDate originalMessageString messageParts
diff --git a/DenbowParser/Script.fsx b/DenbowParser/Script.fsx
@@ -14,9 +14,10 @@ open System.Text.RegularExpressions
 open DenbowParser.Parser
 open DenbowParser.Utils
 
-let message = System.IO.File.ReadAllText("Email.denbow") |> loadMimeMessageFrom 
+let inputString = System.IO.File.ReadAllText("Email.denbow")
+let message = inputString |> loadMimeMessageFrom 
 
-let messageData = messageDataFor message
+let messageData = messageDataFor message inputString
 let messageLines = messageData.MessageLines
 
 let messageParts = parse messageLines PreIntro

diff --git a/DenbowParser/Utils.fs b/DenbowParser/Utils.fs
@@ -8,12 +8,13 @@ open EmailParser.Utils.Text
 open EmailParser.Utils.Date
 open EmailParser.Types
 
-let messageDataFor (message: MimeMessage) =
+let messageDataFor (message: MimeMessage) (originalMessageString: string) =
     let messageLines = htmlPartsOf message |> String.concat "\n" |> toPlainText |> splitIntoLines
     {
         Sender = senderOf message;
         SentDate = dateOf message;
-        MessageLines = messageLines
+        MessageLines = messageLines;
+        EntireMessage = originalMessageString
     }
 
 let startsWithEventDate (line: string) =

diff --git a/MeetupParser/Parser.fs b/MeetupParser/Parser.fs
@@ -82,12 +82,12 @@ let calendarEntryFrom (messageParts: seq<MessagePart>) =
         RsvpLink = (uriFrom rsvp)
     }
 
-let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (messageParts: list<MessagePart>) : EmailData =
+let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (originalMessageString: string) (messageParts: list<MessagePart>) : EmailData =
     let calendarEntries = [ calendarEntryFrom messageParts ]
 
-    { MailDate = sentDate; MailSender = sender; MailIntro = ""; CalendarEntries = calendarEntries }
+    { MailDate = sentDate; MailSender = sender; MailIntro = ""; OriginalMessage = originalMessageString; CalendarEntries = calendarEntries }
 
-let parseMail (message: MimeMessage) : EmailData = 
-    let messageData = messageDataFor message
+let parseMail (message: MimeMessage) (originalMessageString: string) : EmailData = 
+    let messageData = messageDataFor message originalMessageString
     let messageParts = parse messageData.MessageLines Header
-    parseIntoEmailData messageData.Sender messageData.SentDate messageParts
+    parseIntoEmailData messageData.Sender messageData.SentDate originalMessageString messageParts
diff --git a/MeetupParser/Script.fsx b/MeetupParser/Script.fsx
@@ -15,9 +15,10 @@ open EmailParser.Utils.Collections
 open MeetupParser.Utils
 open MeetupParser.Parser
 
-let message = System.IO.File.ReadAllText("Email.meetup") |> loadMimeMessageFrom 
+let inputString = System.IO.File.ReadAllText("Email.meetup")
+let message = inputString |> loadMimeMessageFrom 
 
-let messageData = messageDataFor message
+let messageData = messageDataFor message inputString
 let messageLines = messageData.MessageLines
 
 let message2 = System.IO.File.ReadAllText("Email2.meetup") |> loadMimeMessageFrom 

diff --git a/MeetupParser/Utils.fs b/MeetupParser/Utils.fs
@@ -8,11 +8,12 @@ open EmailParser.Utils.Date
 open EmailParser.Utils.Text
 open EmailParser.Types
 
-let messageDataFor (message: MimeMessage) =
+let messageDataFor (message: MimeMessage) (originalMessageString: string) =
     {
         Sender = senderOf message;
         SentDate = dateOf message;
-        MessageLines = (textOf message |> splitIntoLines)
+        MessageLines = (textOf message |> splitIntoLines);
+        EntireMessage = originalMessageString
     }
 
 let startsWithTitle (line: string) = line.Trim().StartsWith("What:")

diff --git a/ODonnellParser/Parser.fs b/ODonnellParser/Parser.fs
@@ -122,14 +122,14 @@ let rec calendarEntriesFrom (datePart: MessagePart) (messageParts: list<MessageP
         | _ -> sprintf "Nonempty list not starting with date or time part: [%s]" (remainingParts.ToString()) |> failwith
 
 
-let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (messageParts: list<MessagePart>) : EmailData =
+let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (originalMessageString: string) (messageParts: list<MessagePart>) : EmailData =
     let intro = messageParts |> extractWithEmptyStringDefault (function | IntroPart(intro) -> Some(String.concat "\n" intro)  | _ -> None)
     let nonIntroParts = messageParts |> List.filter (function |IntroPart(_) -> false | _ -> true)
     let calendarEntries = calendarEntriesFrom nonIntroParts.Head nonIntroParts.Tail
 
-    { MailDate = sentDate; MailSender = sender; MailIntro = intro; CalendarEntries = calendarEntries }
+    { MailDate = sentDate; MailSender = sender; MailIntro = intro; OriginalMessage = originalMessageString; CalendarEntries = calendarEntries }
 
-let parseMail (message: MimeMessage) : EmailData = 
-    let messageData = messageDataFor message
+let parseMail (message: MimeMessage) (originalMessageString: string) : EmailData = 
+    let messageData = messageDataFor message originalMessageString
     let messageParts = parse messageData.MessageLines PreIntro
-    parseIntoEmailData messageData.Sender messageData.SentDate messageParts
+    parseIntoEmailData messageData.Sender messageData.SentDate originalMessageString messageParts
diff --git a/ODonnellParser/Script.fsx b/ODonnellParser/Script.fsx
@@ -14,9 +14,10 @@ open System.Text.RegularExpressions
 open ODonnellParser.Parser
 open ODonnellParser.Utils
 
-let message = System.IO.File.ReadAllText("Email.odonnell") |> loadMimeMessageFrom
+let inputString = System.IO.File.ReadAllText("Email.odonnell")
+let message = inputString |> loadMimeMessageFrom
 
-let messageData = messageDataFor message
+let messageData = messageDataFor message inputString
 let messageParts = parse messageData.MessageLines PreIntro
 let nonIntroParts = messageParts |> List.filter (function |IntroPart(_) -> false | _ -> true)
 let date = extractDateStringFrom nonIntroParts.Head    

diff --git a/ODonnellParser/Utils.fs b/ODonnellParser/Utils.fs
@@ -8,11 +8,12 @@ open EmailParser.Utils.Date
 open EmailParser.Utils.Text
 open EmailParser.Types
 
-let messageDataFor (message: MimeMessage) =
+let messageDataFor (message: MimeMessage) (originalMessageString: string) =
     {
         Sender = senderOf message;
         SentDate = dateOf message;
-        MessageLines = (textOf message |> splitIntoLines)
+        MessageLines = (textOf message |> splitIntoLines);
+        EntireMessage = originalMessageString
     }
 
 let removeEventUrlFrom (eventHeader: string) = 

diff --git a/ParserTypes/Types.fs b/ParserTypes/Types.fs
@@ -5,7 +5,8 @@ open System
 type EmailMessage = {
     Sender: string;
     SentDate: DateTime;
-    MessageLines: list<string>
+    MessageLines: list<string>;
+    EntireMessage: string
 }
 
 type DateAndTime = {
@@ -25,5 +26,6 @@ type EmailData = {
     MailDate: DateTime;
     MailSender: string;
     MailIntro: string;
+    OriginalMessage: string;
     CalendarEntries: list<CalendarEntry>
 }
diff --git a/Persistence/SQLitePersistence.fs b/Persistence/SQLitePersistence.fs
@@ -19,6 +19,7 @@ let loadEmailData (dataContext: DataContext) emailData =
     email.timestamp <- emailData.MailDate |> secondsSinceEpoch |> Some 
     email.sender <- emailData.MailSender |> Some
     email.intro <- emailData.MailIntro |> Some
+    email.entire_message <- emailData.OriginalMessage |> Some
     dataContext.SubmitUpdates()
     email
 

diff --git a/Persistence/Script.fsx b/Persistence/Script.fsx
@@ -49,6 +49,7 @@ let emailData = {
     MailDate = System.DateTime.Today;
     MailSender = "Charlie O'Donnell <[email protected]>";
     MailIntro = "an\nintro";
+    OriginalMessage = "The Original Message String\n";
     CalendarEntries = [calendarEntry; yesterdayEntry]
 }
 

diff --git a/Persistence/events.sqlitedb b/Persistence/events.sqlitedb
diff --git a/Persistence/schema.ddl b/Persistence/schema.ddl
@@ -3,7 +3,8 @@
    date TEXT,
    timestamp INTEGER, 
    sender TEXT,
-   intro TEXT
+   intro TEXT,
+   entire_message TEXT
 );
 
 CREATE TABLE calendar_entries(