diff --git a/App/App.fsproj b/App/App.fsproj index 5a099d3..7b6b0c4 100644 --- a/App/App.fsproj +++ b/App/App.fsproj @@ -75,6 +75,10 @@ {166E52EB-4E26-4074-A695-63D071AC135F} Persistence + + {561CEF0A-3584-44FE-B5F8-0F574E1019B0} + MeetupParser + diff --git a/App/Program.fs b/App/Program.fs index c88dc59..74d79c9 100644 --- a/App/Program.fs +++ b/App/Program.fs @@ -19,21 +19,23 @@ let usage() = printfn "" printfn " .denbow - extension for Frank Denbow's mails" printfn " .odonnell - extension for Charlie O'Donnell's mails" + printfn " .meetup - extension for mails from meetup.com" printfn "" printfn "Output is written into a file named Events.html in the " printfn "current working directory." let selectParseFunction (fileName: string) = match fileName with - | filename when filename.EndsWith(".denbow") -> DenbowParser.Parser.parseMail - | filename when filename.EndsWith(".odonnell") -> ODonnellParser.Parser.parseMail + | filename when filename.EndsWith(".denbow") -> DenbowParser.Parser.parseMail + | filename when filename.EndsWith(".odonnell") -> ODonnellParser.Parser.parseMail + | filename when filename.EndsWith(".meetup") -> MeetupParser.Parser.parseMail | _ -> sprintf "Unrecognized file extension for file: %s" fileName |> failwith let loadDataFrom (filename: string) = let inputString = System.IO.File.ReadAllText(filename) let message = loadMimeMessageFrom(inputString) let parseFunction = selectParseFunction filename - let parsed = parseFunction message + let parsed = parseFunction message inputString loadMail parsed () diff --git a/DenbowParser/Parser.fs b/DenbowParser/Parser.fs index 45f020a..1498afb 100644 --- a/DenbowParser/Parser.fs +++ b/DenbowParser/Parser.fs @@ -131,14 +131,14 @@ let rec calendarEntriesFrom (messageParts: list) : list calendarEntry :: (calendarEntriesFrom remainingParts) | _ -> sprintf "Nonempty list not starting with RSVP link part: [%s]" (remainingParts.ToString()) |> failwith -let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (messageParts: list) : EmailData = +let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (originalMessageString: string) (messageParts: list) : EmailData = let intro = messageParts |> extractWithEmptyStringDefault (function | IntroPart(intro) -> Some(String.concat "\n" intro) | _ -> None) let nonIntroParts = messageParts |> List.filter (function |IntroPart(_) -> false | _ -> true) let calendarEntries = calendarEntriesFrom nonIntroParts - { MailDate = sentDate; MailSender = sender; MailIntro = intro; CalendarEntries = calendarEntries } + { MailDate = sentDate; MailSender = sender; MailIntro = intro; OriginalMessage = originalMessageString; CalendarEntries = calendarEntries } -let parseMail (message: MimeMessage) : EmailData = - let messageData = messageDataFor message +let parseMail (message: MimeMessage) (originalMessageString: string) : EmailData = + let messageData = messageDataFor message originalMessageString let messageParts = parse messageData.MessageLines PreIntro - parseIntoEmailData messageData.Sender messageData.SentDate messageParts \ No newline at end of file + parseIntoEmailData messageData.Sender messageData.SentDate originalMessageString messageParts \ No newline at end of file diff --git a/DenbowParser/Script.fsx b/DenbowParser/Script.fsx index cd1eaa7..2357c98 100644 --- a/DenbowParser/Script.fsx +++ b/DenbowParser/Script.fsx @@ -14,9 +14,10 @@ open System.Text.RegularExpressions open DenbowParser.Parser open DenbowParser.Utils -let message = System.IO.File.ReadAllText("Email.denbow") |> loadMimeMessageFrom +let inputString = System.IO.File.ReadAllText("Email.denbow") +let message = inputString |> loadMimeMessageFrom -let messageData = messageDataFor message +let messageData = messageDataFor message inputString let messageLines = messageData.MessageLines let messageParts = parse messageLines PreIntro diff --git a/DenbowParser/Utils.fs b/DenbowParser/Utils.fs index d7ce710..ade758e 100644 --- a/DenbowParser/Utils.fs +++ b/DenbowParser/Utils.fs @@ -8,12 +8,13 @@ open EmailParser.Utils.Text open EmailParser.Utils.Date open EmailParser.Types -let messageDataFor (message: MimeMessage) = +let messageDataFor (message: MimeMessage) (originalMessageString: string) = let messageLines = htmlPartsOf message |> String.concat "\n" |> toPlainText |> splitIntoLines { Sender = senderOf message; SentDate = dateOf message; - MessageLines = messageLines + MessageLines = messageLines; + EntireMessage = originalMessageString } let startsWithEventDate (line: string) = @@ -48,7 +49,7 @@ let extractTitleFrom (eventHeader: string) = eventHeader.Substring(startIndex).Trim() let dateAndTimeFrom (dateTimeString: string) = - let normalized = dateTimeString |> regexReplace " +" " " + let normalized = dateTimeString |> normalizeSpace |> regexReplace @"\s+:" ":" //No space around colons in time |> regexReplace @":\s+" ":" |> regexReplaceIgnoreCase @"\s+am\s+" "am " //No space before am or pm @@ -67,7 +68,7 @@ let dateAndTimeFrom (dateTimeString: string) = let containsCalendarLink (descriptionLine: string) = - let normalizedLine = descriptionLine.ToLower() |> regexReplace " +" " " + let normalizedLine = descriptionLine.ToLower() |> normalizeSpace normalizedLine.Contains("view in calendar") let removeCalendarLink (descriptionLine: string) = diff --git a/EmailParser.sln b/EmailParser.sln index 4797882..38d9c5d 100644 --- a/EmailParser.sln +++ b/EmailParser.sln @@ -15,6 +15,8 @@ Project("{f2a71f9b-5d33-465a-a702-920d77279786}") = "DenbowParser", "DenbowParse EndProject Project("{f2a71f9b-5d33-465a-a702-920d77279786}") = "Utils", "Utils\Utils.fsproj", "{EFAA7888-F46F-4E5B-9020-B03B90422E6F}" EndProject +Project("{f2a71f9b-5d33-465a-a702-920d77279786}") = "MeetupParser", "MeetupParser\MeetupParser.fsproj", "{561CEF0A-3584-44FE-B5F8-0F574E1019B0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -33,6 +35,10 @@ Global {427B4B20-D0A6-4AA9-933A-F7D23F76DF4E}.Debug|Any CPU.Build.0 = Debug|Any CPU {427B4B20-D0A6-4AA9-933A-F7D23F76DF4E}.Release|Any CPU.ActiveCfg = Release|Any CPU {427B4B20-D0A6-4AA9-933A-F7D23F76DF4E}.Release|Any CPU.Build.0 = Release|Any CPU + {561CEF0A-3584-44FE-B5F8-0F574E1019B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {561CEF0A-3584-44FE-B5F8-0F574E1019B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {561CEF0A-3584-44FE-B5F8-0F574E1019B0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {561CEF0A-3584-44FE-B5F8-0F574E1019B0}.Release|Any CPU.Build.0 = Release|Any CPU {59D4D8A6-CCF5-449E-A5FC-00078D897A3A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {59D4D8A6-CCF5-449E-A5FC-00078D897A3A}.Debug|Any CPU.Build.0 = Debug|Any CPU {59D4D8A6-CCF5-449E-A5FC-00078D897A3A}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/EmailParser.userprefs b/EmailParser.userprefs index 3a3f782..8f19828 100644 --- a/EmailParser.userprefs +++ b/EmailParser.userprefs @@ -1,6 +1,10 @@  - + + + + + diff --git a/MeetupParser/Email.meetup b/MeetupParser/Email.meetup new file mode 100644 index 0000000..27ea869 --- /dev/null +++ b/MeetupParser/Email.meetup @@ -0,0 +1,407 @@ + +Delivered-To: pblair@cyrusinnovation.com +Received: by 10.43.48.2 with SMTP id uu2csp236872icb; + Mon, 8 Jul 2013 09:50:56 -0700 (PDT) +X-Received: by 10.49.24.52 with SMTP id r20mr16841813qef.54.1373302256739; + Mon, 08 Jul 2013 09:50:56 -0700 (PDT) +Return-Path: +Received: from mail0.phi.meetup.com (mail0.phi.meetup.com. [38.123.132.110]) + by mx.google.com with ESMTP id b4si6273145qan.114.2013.07.08.09.50.56 + for ; + Mon, 08 Jul 2013 09:50:56 -0700 (PDT) +Received-SPF: pass (google.com: domain of info@meetup.com designates 38.123.132.110 as permitted sender) client-ip=38.123.132.110; +Authentication-Results: mx.google.com; + spf=pass (google.com: domain of info@meetup.com designates 38.123.132.110 as permitted sender) smtp.mail=info@meetup.com +DomainKey-Signature: a=rsa-sha1; c=nofws; q=dns; s=meetup; d=meetup.com; + b=U4NCda/ahhgCvtPDaIXZGJBDdIKX9wlo5CaWVURHZJudxUVCTcV7/5+U7JTogIgoqnDFlHIFMKAG + 4jNYNUjRV1ZT42zVZFumd0MXnyyXf44Znwsw0PsnpLbrlQjnZtkXtw9I99/0b8oAefUl1isuhgYA + AHiw7rndhCvVBbHKzn0=; +Received: from jobs0.meetup.com (10.3.0.30) by mail0.phi.meetup.com id hrbmv015odg9 for ; Mon, 8 Jul 2013 12:50:55 -0400 (envelope-from ) +Message-ID: <469868696.1373302255705.JavaMail.root@jobs0.meetup.com> +From: eXtreme Tuesday Club - Boston +Sender: info@meetup.com +To: pblair@cyrusinnovation.com +Subject: Tomorrow: Join 3 Extreme Programmers at "eXtreme Tuesday Club - + Boston Monthly Meetup" +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_Part_83753311_1621859833.1373302255704" +X-MEETUP-MESG-ID: 34080162 +X-MEETUP-RECIP-ID: 5342366 +X-MEETUP-TRACK: md2 +Date: Mon, 8 Jul 2013 12:50:55 -0400 + +------=_Part_83753311_1621859833.1373302255704 +Content-Type: multipart/alternative; + boundary="----=_Part_83753312_588281860.1373302255704" + +------=_Part_83753312_588281860.1373302255704 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Just a quick reminder that eXtreme Tuesday Club - Boston has a Meetup tomorrow. Are you going? + +Don Roby added this Meetup for eXtreme Tuesday Club - Boston + +What: eXtreme Tuesday Club - Boston Monthly Meetup + +When: Tuesday, July 9, 2013 6:30 PM + +Where: +Cambridge Brewing Company +1 Kendall Square +Cambridge, MA + +We're moving the meetings back to Tuesdays, and changing the name back. + +Click here to say you're going http://www.meetup.com/xp-28/events/125626492/t/md2_rt + +To stop receiving reminder emails about Meetups from this group, click here: +http://www.meetup.com/__ms5342366/xp-28/optout/?submit=true&email=evRemind&expires=1373475055691&sig=2e84daf5c3cc5a3b90dfa30c8e5fe91b55a7a0f2 + +-- +Add info@meetup.com to your address book to receive all Meetup emails + +To manage your email settings for this group, go to: http://www.meetup.com/xp-28/settings/ +Meetup, POB 4668 #37895 NY NY USA 10163 + +Meetup HQ in NYC is hiring! http://www.meetup.com/jobs/ + +------=_Part_83753312_588281860.1373302255704 +Content-Type: text/html; charset="UTF-8" +Content-Transfer-Encoding: 7bit + + + + + + + + + + + + + + + + +Meetup + + + +
+ + + + + + + +
 
Meetup
 
+ +
+ +
+tomorrow +
+ +
eXtreme Tuesday Club - Boston
+ +
+ +
+
+Tuesday, July 9, 2013
6:30 PM +
+
+Cambridge Brewing Company
+1 Kendall Square
+Cambridge, MA 02139
+
+ +
+ + + + + + + + + + +
+
Will you attend?
+
+
+ + + + + +
+ + + +
+
+
+
+ +
+ + +
+
+ +
+
+
+ + +3 Extreme Programmers attending, including: + + +
+
+
+ + + +
+
+ +
+ 		+
+ Don Roby +
+
"I'm a software developer with some experience in XP, and an interest in agile processes in general."
+
+
+
+ + + +
+
+ +
+ 		+ +
+
+
+
+
+ + + +
+
+We're moving the meetings back to Tuesdays, and changing the name back.
+Learn more +
+
+ + + +
+ + + + + + +
+
+
Aug
+
13
+
+ 		+ + + + + +
+ eXtreme Tuesday Club - Boston Monthly Meetup
+
+ Tuesday, August 13, 2013 6:30 PM + · + 1 attending +
+ 		+ +
+
+
+ + + +
+
+ +
+ +
+ +
+To stop receiving reminder emails about Meetups from this group, click here. +
+
+ + +
+
+
+

Add info@meetup.com to your address book to receive all Meetup emails

+

To manage your email settings, click here

+

Meetup, POB 4668 #37895 NY NY USA 10163

+

Meetup HQ in NYC is hiring! meetup.com/jobs

+
+
+
+ +
+ + +------=_Part_83753312_588281860.1373302255704-- + +------=_Part_83753311_1621859833.1373302255704 +Content-Type: text/calendar; name=meetup.ics; method=REQUEST; charset=UTF-8 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename=meetup.ics + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Meetup//Meetup Events v1.0//EN +CALSCALE:GREGORIAN +METHOD:PUBLISH +X-WR-CALNAME:Meetup Events +X-MS-OLK-FORCEINSPECTOROPEN:TRUE +BEGIN:VTIMEZONE +TZID:America/New_York +X-LIC-LOCATION:America/New_York +BEGIN:DAYLIGHT +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +TZNAME:EDT +DTSTART:19700308T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +TZNAME:EST +DTSTART:19701101T020000 +RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +DTSTAMP:20130708T164227Z +DTSTART;TZID=America/New_York:20130709T183000 +DTEND;TZID=America/New_York:20130709T213000 +STATUS:CONFIRMED +SUMMARY:eXtreme Tuesday Club - Boston Monthly Meetup +DESCRIPTION:eXtreme Tuesday Club - Boston\nTuesday\, July 9 at 6:30 PM\n\ + nWe're moving the meetings back to Tuesdays\, and changing the name back + .\n\nDetails: http://www.meetup.com/xp-28/events/125626492/ +ORGANIZER;CN=Meetup Reminder:MAILTO:info@meetup.com +CLASS:PUBLIC +CREATED:20101226T183523Z +GEO:42.36;-71.06 +LOCATION:Cambridge Brewing Company (1 Kendall Square\, Cambridge\, MA 021 + 39) +URL:http://www.meetup.com/xp-28/events/125626492/ +LAST-MODIFIED:20101226T183523Z +UID:event_dngcqyrkbmb@meetup.com +END:VEVENT +END:VCALENDAR + +------=_Part_83753311_1621859833.1373302255704-- + diff --git a/MeetupParser/Email2.meetup b/MeetupParser/Email2.meetup new file mode 100644 index 0000000..a03e69e --- /dev/null +++ b/MeetupParser/Email2.meetup @@ -0,0 +1,508 @@ + +Delivered-To: psfblair@gmail.com +Received: by 10.194.55.98 with SMTP id r2csp174890wjp; + Tue, 24 Jun 2014 10:08:15 -0700 (PDT) +X-Received: by 10.66.163.164 with SMTP id yj4mr3218016pab.91.1403629694978; + Tue, 24 Jun 2014 10:08:14 -0700 (PDT) +Return-Path: +Received: from mxfwd2.rollernet.us (mxfwd2.rollernet.us. [2607:fe70:0:16::b]) + by mx.google.com with ESMTPS id mk6si1287886pab.91.2014.06.24.10.08.14 + for + (version=TLSv1 cipher=RC4-SHA bits=128/128); + Tue, 24 Jun 2014 10:08:14 -0700 (PDT) +Received-SPF: pass (google.com: domain of SRS0+HsA7=3V=meetup.com=info@mxfwd2.rollernet.us designates 2607:fe70:0:16::b as permitted sender) client-ip=2607:fe70:0:16::b; +Authentication-Results: mx.google.com; + spf=pass (google.com: domain of SRS0+HsA7=3V=meetup.com=info@mxfwd2.rollernet.us designates 2607:fe70:0:16::b as permitted sender) smtp.mail=SRS0+HsA7=3V=meetup.com=info@mxfwd2.rollernet.us +Received: from mail2.rollernet.us (mail2.rollernet.us [208.79.241.2]) + by mxfwd2.rollernet.us (Postfix) with ESMTP id D170BC02A8BD + for ; Tue, 24 Jun 2014 10:08:13 -0700 (PDT) +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on mail2.rollernet.us +X-Spam-Level: *** +X-Spam-Status: No, score=3.7 required=5.0 tests=DKIM_SIGNED,DKIM_VALID, + DKIM_VALID_AU,HTML_MESSAGE,LONG_IMG_URI,T_RP_MATCHES_RCVD autolearn=disabled + version=3.3.2 +X-Envelope-From: info@meetup.com +Received: from mail5.nyi.meetup.com (mail5.nyi.meetup.com [64.90.170.35]) + by mail2.rollernet.us (Postfix) with ESMTP + for ; Tue, 24 Jun 2014 10:08:10 -0700 (PDT) +DomainKey-Signature: a=rsa-sha1; c=nofws; q=dns; s=meetup; d=meetup.com; + b=BN9fznChOq4IbUESpeN8tJxRZ/GTyBvH4OQ48ukM8FR5x9AS7i0bnE/I8IKUJw7at253+1ej/lB3 + TjbmQgVK91qpjzB84uBdQx4jmiwlyqdDj7/yhCSTbmwRcLGU+jFCH+eGFiNqZw8qN2dCoDDT0ZYf + hQidHKbyg3SHvGLtAgY=; +Received: by mail5.nyi.meetup.com id hl6o7m15obo4 for ; Tue, 24 Jun 2014 13:08:06 -0400 (envelope-from ) +Received: by pmta0.phi.meetup.com id hl6o7c15odgg for ; Tue, 24 Jun 2014 13:08:06 -0400 (envelope-from ) +Message-ID: <620576586.1403629686054.JavaMail.root@jobs0.meetup.com> +From: Agile / Lean Practitioners +Sender: info@meetup.com +To: ciriwe@phobot.net +Subject: Tomorrow: Join 69 Agile / Lean Practitioners at "Progress at human + scale: how discipline, patience & respect leads to org change" +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_Part_12790036_136585416.1403629686052" +X-MEETUP-MESG-ID: 80482642 +X-MEETUP-RECIP-ID: 6387787 +X-MEETUP-TRACK: md1 +Date: Tue, 24 Jun 2014 13:08:06 -0400 +Received-SPF: pass (meetup.com: 64.90.170.35 is authorized to use 'info@meetup.com' in 'mfrom' identity (mechanism 'ip4:64.90.170.0/26' matched)) receiver=mail2.rollernet.us; identity=mailfrom; envelope-from="info@meetup.com"; helo=mail5.nyi.meetup.com; client-ip=64.90.170.35 +X-Rollernet-Abuse: Processed by Roller Network Mail Services. See our abuse policy at http://www.rollernet.us/policy +X-Rollernet-Tracking: Tracking ID 67b3.53a9b07a.49e9c.0 +X-Rollernet-FwdTo: default->psfblair@gmail.com + +------=_Part_12790036_136585416.1403629686052 +Content-Type: multipart/alternative; + boundary="----=_Part_12790038_1698265910.1403629686053" + +------=_Part_12790038_1698265910.1403629686053 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit + +Just a quick reminder that Agile / Lean Practitioners has a Meetup tomorrow. Are you going? + +Terrence McGovern added this Meetup for Agile / Lean Practitioners + +What: Progress at human scale: how discipline, patience & respect leads to org change + +When: Wednesday, June 25, 2014 6:30 PM + +Where: +Kaplan Test Prep +395 Hudson Street, 3rd Floor +New York, NY + +There are moments an organization recognizes a need for change, calls upon outside expertise and dives in with commitment. This is what you do while waiting for that moment. This is about a team changing from within at a pace and to the degree people... + +Click here to say you're going http://www.meetup.com/agile-lean-practitioners/events/181749242/t/md1_rt + +To stop receiving reminder emails about Meetups from this group, click here: +http://www.meetup.com/__ms6387787/agile-lean-practitioners/optout/?submit=true&_ms_unsub=true&email=evRemind&expires=1403802486033&sig=952a917c4ffe4d50dccc262cb78e433c2befa381 + +-- +Add info@meetup.com to your address book to receive all Meetup emails + +To manage your email settings for this group, go to: http://www.meetup.com/agile-lean-practitioners/settings/ +Meetup, POB 4668 #37895 NY NY USA 10163 + +Meetup HQ in NYC is hiring! http://www.meetup.com/jobs/ + +------=_Part_12790038_1698265910.1403629686053 +Content-Type: text/html; charset="UTF-8" +Content-Transfer-Encoding: 7bit + + + + + + + + + + + + + + + + + +Meetup + + + +
+ + + + + + + +
 
 +Meetup +
 
+ +
+ +
+tomorrow +
+ +
Agile / Lean Practitioners
+ +
+ +
+
+Wednesday, June 25, 2014
6:30 PM +
+
+Kaplan Test Prep
+395 Hudson Street, 3rd Floor
+New York, NY 10014
+
+ +
+ + + + + + + + + + +
+
Are you going?
+
+
+ + + + + +
+ + + +
+
+
+
+ +
+ + +
+
+ +
+
+
+ + +69 Agile / Lean Practitioners going, including: + + +
+
+
+ + + +
+
+ +
+ 		+ +
"Product "
+
+
+
+ + + +
+
+ +
+ 		+ +
"Investor and software entrepreneur [2 successful enterprise software startups sold]. Launching an investment and technology website. +"
+
+
+
+
+ + + +
+
+ +
+ 		+
+ Dave +
+
"Hi. I work for a web/mobile dev company"
+
+
+
+ + + +
+
+ +
+ 		+ +
"Hi, I'm a software engineer at Kaplan Test Prep"
+
+
+
+
+ + + +
+
+ +
+ 		+
+ Andrew +
+
"Java/J2EE developer"
+
+
+
+ + + +
+
+ +
+ 		+ +
"Scrum Master, Lean enthusiast, recovering entrepreneur, now @netatwork_corp"
+
+
+
+
+
+
+ + + +
+
+There are moments an organization recognizes a need for change, calls upon outside expertise and dives in with commitment. This is what you do while waiting for that moment. This is about a team changing from within at a pace and to the degree people...
+Learn more +
+
+ + + +
+ + + + + + + + + + +
+
+
Jul
+
16
+
+ 		+ + + + + +
+ Moving from Agile Software Dev to Business Agility and Radical Innovation
+
+ Wednesday, July 16, 2014 6:30 PM + · + 52 attending +
+ 		+ +
+
+
+
Aug
+
20
+
+ 		+ + + + + +
+ Thinking, Talking, Showing; The Three Ways of Sketching
+
+ Wednesday, August 20, 2014 6:30 PM + · + 49 attending +
+ 		+ +
+
+
+ + + + +
+ +
+ + +
+
+
+

Unsubscribe from similar emails from this Meetup Group

+

Add info@meetup.com to your address book to receive all Meetup emails

+

Meetup, POB 4668 #37895 NY NY USA 10163

+

Meetup HQ in NYC is hiring! meetup.com/jobs

+
+
+
+ +
+ + +------=_Part_12790038_1698265910.1403629686053-- + +------=_Part_12790036_136585416.1403629686052 +Content-Type: text/calendar; name=meetup.ics; method=REQUEST; charset=UTF-8 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename=meetup.ics + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Meetup//Meetup Events v1.0//EN +CALSCALE:GREGORIAN +METHOD:PUBLISH +X-WR-CALNAME:Events - Agile / Lean Practitioners +X-MS-OLK-FORCEINSPECTOROPEN:TRUE +BEGIN:VTIMEZONE +TZID:America/New_York +X-LIC-LOCATION:America/New_York +BEGIN:DAYLIGHT +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +TZNAME:EDT +DTSTART:19700308T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +TZNAME:EST +DTSTART:19701101T020000 +RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +DTSTAMP:20140624T165549Z +DTSTART;TZID=America/New_York:20140625T183000 +DTEND;TZID=America/New_York:20140625T200000 +STATUS:CONFIRMED +SUMMARY:Progress at human scale: how discipline\, patience & respect lead + s to org change +DESCRIPTION:Agile / Lean Practitioners\nWednesday\, June 25 at 6:30 PM\n\ + nThere are moments an organization recognizes a need for change\, calls + upon outside expertise and dives in with commitment. This is what you do + while w...\n\nDetails: http://www.meetup.com/agile-lean-practitioners/e + vents/181749242/ +ORGANIZER;CN=Meetup Reminder:MAILTO:info@meetup.com +CLASS:PUBLIC +CREATED:20140507T193628Z +GEO:40.72;-74.01 +LOCATION:Kaplan Test Prep (395 Hudson Street\, 3rd Floor\, New York\, NY + 10014) +URL:http://www.meetup.com/agile-lean-practitioners/events/181749242/ +SEQUENCE:1 +LAST-MODIFIED:20140508T210404Z +UID:event_181749242@meetup.com +END:VEVENT +END:VCALENDAR + +------=_Part_12790036_136585416.1403629686052-- + diff --git a/MeetupParser/MeetupParser.fsproj b/MeetupParser/MeetupParser.fsproj new file mode 100644 index 0000000..bb7aefb --- /dev/null +++ b/MeetupParser/MeetupParser.fsproj @@ -0,0 +1,72 @@ + + + + Debug + AnyCPU + {561CEF0A-3584-44FE-B5F8-0F574E1019B0} + Library + MeetupParser + MeetupParser + v4.5 + + + true + full + false + bin\Debug + DEBUG + prompt + false + false + + + + + true + pdbonly + true + bin\Release + prompt + + + false + true + + + + + + + + + ..\packages\MimeKitLite.0.36.0.0\lib\net40\MimeKitLite.dll + + + + + + + + + + + + + + {D7E39B94-578A-43D2-9FCE-37D8898B6614} + ParserTypes + + + {EFAA7888-F46F-4E5B-9020-B03B90422E6F} + Utils + + + + + PreserveNewest + + + PreserveNewest + + + \ No newline at end of file diff --git a/MeetupParser/Parser.fs b/MeetupParser/Parser.fs new file mode 100644 index 0000000..9020564 --- /dev/null +++ b/MeetupParser/Parser.fs @@ -0,0 +1,93 @@ +module MeetupParser.Parser + +open MimeKit +open EmailParser.Types +open EmailParser.Utils.Collections +open EmailParser.Utils.Text +open EmailParser.Utils.Uri + +open MeetupParser.Utils + + +type State = + | Header + | EventTitle + | EventDateTime + | EventLocation + | EventDescription + | RSVPLink + | MessageTrailer + +type MessagePart = + | TitlePart of string + | DateTimePart of string + | LocationPart of list + | DescriptionPart of list + | RsvpLinkPart of string + +let rec parse (mail: seq) (state: State) : list = + let mailText = List.ofSeq mail + match state with + | Header -> header mailText + | EventTitle -> eventTitle mailText + | EventDateTime -> eventDateTime mailText + | EventLocation -> eventLocation mailText + | EventDescription -> eventDescription mailText + | RSVPLink -> rsvpLink mailText + | MessageTrailer -> messageTrailer mailText + +and header (mailText: list) = + let headerLines, rest = mailText |> takeAndSkipUntil startsWithTitle + (parse rest EventTitle) + +and eventTitle (mailText: list) = + let title = mailText.Head |> extractTitleFrom |> TitlePart + let ignore, rest = mailText |> takeAndSkipUntil startsWithEventDate + title :: (parse rest EventDateTime) + +and eventDateTime (mailText: list) = + let date = mailText.Head |> extractDateTimeFrom |> DateTimePart + let ignore, rest = mailText |> takeAndSkipUntil startsWithLocation + date :: (parse rest EventLocation) + +and eventLocation (mailText: list) = + let locationLines, rest = mailText.Tail |> takeAndSkipUntil isBlank + let location = locationLines |> List.ofSeq |> LocationPart + location :: (parse rest EventDescription) + +and eventDescription (mailText: list) = + let descriptionLines, rest = mailText.Tail |> takeAndSkipUntil isRsvpLine + let description = descriptionLines |> List.ofSeq |> DescriptionPart + description :: (parse rest RSVPLink) + +and rsvpLink (mailText: list) = + let link = mailText.Head |> extractRsvpLinkFrom |> RsvpLinkPart + link :: (parse mailText.Tail MessageTrailer) + +and messageTrailer (mailText: list) = [] + +let calendarEntryFrom (messageParts: seq) = + + let title = messageParts |> extractWithEmptyStringDefault (function | TitlePart(aTitle) -> Some(aTitle) | _ -> None) + let date = messageParts |> extractWithEmptyStringDefault (function | DateTimePart(dateTime) -> Some(dateTime) | _ -> None) + let location = messageParts |> extractWithEmptyStringDefault (function | LocationPart(loc) -> Some(String.concat "\n" loc) | _ -> None) + let description = messageParts |> extractWithEmptyStringDefault (function | DescriptionPart(desc) -> Some(String.concat "\n" desc) | _ -> None) + let rsvp = messageParts |> extractWithEmptyStringDefault (function | RsvpLinkPart(link) -> Some(link) | _ -> None) + + { + EventDate = (dateAndTimeFrom date); + EventTitle = title; + EventLocation = Some(location); + EventDescription = description.Trim(); + RsvpLink = (uriFrom rsvp) + } + +let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (originalMessageString: string) (messageParts: list) : EmailData = + let calendarEntries = [ calendarEntryFrom messageParts ] + + { MailDate = sentDate; MailSender = sender; MailIntro = ""; OriginalMessage = originalMessageString; CalendarEntries = calendarEntries } + +let parseMail (message: MimeMessage) (originalMessageString: string) : EmailData = + let messageData = messageDataFor message originalMessageString + let messageParts = parse messageData.MessageLines Header + parseIntoEmailData messageData.Sender messageData.SentDate originalMessageString messageParts \ No newline at end of file diff --git a/MeetupParser/Script.fsx b/MeetupParser/Script.fsx new file mode 100644 index 0000000..fc32265 --- /dev/null +++ b/MeetupParser/Script.fsx @@ -0,0 +1,28 @@ +#I "bin/Debug/" +#r "ParserTypes.dll" +#r "EmailParserUtils.dll" +#r "MimeKitLite.dll" + +System.Environment.CurrentDirectory <- __SOURCE_DIRECTORY__ + +#load "Utils.fs" +#load "Parser.fs" + +open EmailParser.Utils.Mime +open EmailParser.Utils.Text +open EmailParser.Utils.Collections + +open MeetupParser.Utils +open MeetupParser.Parser + +let inputString = System.IO.File.ReadAllText("Email.meetup") +let message = inputString |> loadMimeMessageFrom + +let messageData = messageDataFor message inputString +let messageLines = messageData.MessageLines + +let message2 = System.IO.File.ReadAllText("Email2.meetup") |> loadMimeMessageFrom +let message2Data = messageDataFor message2 + +let messageParts = parse messageLines Header +let parsed = parseMail message \ No newline at end of file diff --git a/MeetupParser/Utils.fs b/MeetupParser/Utils.fs new file mode 100644 index 0000000..f2501cf --- /dev/null +++ b/MeetupParser/Utils.fs @@ -0,0 +1,62 @@ +module MeetupParser.Utils + +open System +open System.Text.RegularExpressions +open MimeKit +open EmailParser.Utils.Mime +open EmailParser.Utils.Date +open EmailParser.Utils.Text +open EmailParser.Types + +let messageDataFor (message: MimeMessage) (originalMessageString: string) = + { + Sender = senderOf message; + SentDate = dateOf message; + MessageLines = (textOf message |> splitIntoLines); + EntireMessage = originalMessageString + } + +let startsWithTitle (line: string) = line.Trim().StartsWith("What:") + +let extractTitleFrom (titleLine: string) = + let startIndex = titleLine.IndexOf(": ") + 2 + titleLine.Substring(startIndex).Trim() + +let startsWithEventDate (line: string) = line.Trim().StartsWith("When:") + +let extractDateTimeFrom (dateTimeLine: string) = + let startIndex = dateTimeLine.IndexOf(": ") + 2 + dateTimeLine.Substring(startIndex).Trim() + +let startsWithLocation (line: string) = line.Trim().StartsWith("Where:") + +let isRsvpLine (line: string) = line.Trim() |> normalizeSpace |> (fun str -> str.StartsWith("Click here to say")) + +let extractRsvpLinkFrom (rsvpLine: string) = + let startIndex = rsvpLine.IndexOf("http") + rsvpLine.Substring(startIndex).Trim() + +let dateFrom = function + | dayOfWeek :: month :: day :: year :: rest when isDayOfWeek dayOfWeek && isMonth month && isYear year -> dateFromMonthDayYear month day year + | dayOfWeek :: month :: day :: rest when isDayOfWeek dayOfWeek && isMonth month -> dateFromMonthDay month day + | month :: day :: year :: rest when isMonth month && isYear year -> dateFromMonthDayYear month day year + | month :: day :: rest when isMonth month -> dateFromMonthDay month day + | other -> failwith(sprintf "unable to parse date: %A" other) + +let dateAndTimeFrom (dateTimeString: string) = + let normalized = dateTimeString.Trim() |> normalizeSpace + |> regexReplace @"\s+:" ":" //No space around colons in time + |> regexReplace @":\s+" ":" + |> regexReplace @"\s+," "," //No space before commas + |> regexReplaceIgnoreCase @"\s+am\s*" "am" //No space before am or pm + |> regexReplaceIgnoreCase @"\s+pm\s*" "pm" + + let parts = normalized.Split(' ') + + let date = List.ofArray parts |> dateFrom + + match hoursAndMinutesFrom parts.[parts.Length - 1] with + | Some(hours, minutes) -> + let dateTime = date.AddHours(hours).AddMinutes(minutes) + { Date = dateTime ; Time = Some(dateTime) } + | None -> { Date = date ; Time = None } diff --git a/MeetupParser/packages.config b/MeetupParser/packages.config new file mode 100644 index 0000000..c18b875 --- /dev/null +++ b/MeetupParser/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/ODonnellParser/Parser.fs b/ODonnellParser/Parser.fs index c39cd95..e70b846 100644 --- a/ODonnellParser/Parser.fs +++ b/ODonnellParser/Parser.fs @@ -122,14 +122,14 @@ let rec calendarEntriesFrom (datePart: MessagePart) (messageParts: list sprintf "Nonempty list not starting with date or time part: [%s]" (remainingParts.ToString()) |> failwith -let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (messageParts: list) : EmailData = +let parseIntoEmailData (sender: string) (sentDate: System.DateTime) (originalMessageString: string) (messageParts: list) : EmailData = let intro = messageParts |> extractWithEmptyStringDefault (function | IntroPart(intro) -> Some(String.concat "\n" intro) | _ -> None) let nonIntroParts = messageParts |> List.filter (function |IntroPart(_) -> false | _ -> true) let calendarEntries = calendarEntriesFrom nonIntroParts.Head nonIntroParts.Tail - { MailDate = sentDate; MailSender = sender; MailIntro = intro; CalendarEntries = calendarEntries } + { MailDate = sentDate; MailSender = sender; MailIntro = intro; OriginalMessage = originalMessageString; CalendarEntries = calendarEntries } -let parseMail (message: MimeMessage) : EmailData = - let messageData = messageDataFor message +let parseMail (message: MimeMessage) (originalMessageString: string) : EmailData = + let messageData = messageDataFor message originalMessageString let messageParts = parse messageData.MessageLines PreIntro - parseIntoEmailData messageData.Sender messageData.SentDate messageParts \ No newline at end of file + parseIntoEmailData messageData.Sender messageData.SentDate originalMessageString messageParts \ No newline at end of file diff --git a/ODonnellParser/Script.fsx b/ODonnellParser/Script.fsx index 2bcdd68..fcded6c 100755 --- a/ODonnellParser/Script.fsx +++ b/ODonnellParser/Script.fsx @@ -14,9 +14,10 @@ open System.Text.RegularExpressions open ODonnellParser.Parser open ODonnellParser.Utils -let message = System.IO.File.ReadAllText("Email.odonnell") |> loadMimeMessageFrom +let inputString = System.IO.File.ReadAllText("Email.odonnell") +let message = inputString |> loadMimeMessageFrom -let messageData = messageDataFor message +let messageData = messageDataFor message inputString let messageParts = parse messageData.MessageLines PreIntro let nonIntroParts = messageParts |> List.filter (function |IntroPart(_) -> false | _ -> true) let date = extractDateStringFrom nonIntroParts.Head diff --git a/ODonnellParser/Utils.fs b/ODonnellParser/Utils.fs index 65d23a3..c9b0822 100644 --- a/ODonnellParser/Utils.fs +++ b/ODonnellParser/Utils.fs @@ -8,11 +8,12 @@ open EmailParser.Utils.Date open EmailParser.Utils.Text open EmailParser.Types -let messageDataFor (message: MimeMessage) = +let messageDataFor (message: MimeMessage) (originalMessageString: string) = { Sender = senderOf message; SentDate = dateOf message; - MessageLines = (textOf message |> splitIntoLines) + MessageLines = (textOf message |> splitIntoLines); + EntireMessage = originalMessageString } let removeEventUrlFrom (eventHeader: string) = diff --git a/ParserTypes/Types.fs b/ParserTypes/Types.fs index dfed13c..2a58bed 100644 --- a/ParserTypes/Types.fs +++ b/ParserTypes/Types.fs @@ -5,7 +5,8 @@ open System type EmailMessage = { Sender: string; SentDate: DateTime; - MessageLines: list + MessageLines: list; + EntireMessage: string } type DateAndTime = { @@ -25,5 +26,6 @@ type EmailData = { MailDate: DateTime; MailSender: string; MailIntro: string; + OriginalMessage: string; CalendarEntries: list } \ No newline at end of file diff --git a/Persistence/SQLitePersistence.fs b/Persistence/SQLitePersistence.fs index 5c28e68..1ae5cad 100644 --- a/Persistence/SQLitePersistence.fs +++ b/Persistence/SQLitePersistence.fs @@ -19,6 +19,7 @@ let loadEmailData (dataContext: DataContext) emailData = email.timestamp <- emailData.MailDate |> secondsSinceEpoch |> Some email.sender <- emailData.MailSender |> Some email.intro <- emailData.MailIntro |> Some + email.entire_message <- emailData.OriginalMessage |> Some dataContext.SubmitUpdates() email diff --git a/Persistence/Script.fsx b/Persistence/Script.fsx index 0be0eb9..07e8082 100644 --- a/Persistence/Script.fsx +++ b/Persistence/Script.fsx @@ -49,6 +49,7 @@ let emailData = { MailDate = System.DateTime.Today; MailSender = "Charlie O'Donnell "; MailIntro = "an\nintro"; + OriginalMessage = "The Original Message String\n"; CalendarEntries = [calendarEntry; yesterdayEntry] } diff --git a/Persistence/events.sqlitedb b/Persistence/events.sqlitedb index ad6e484..b3cbfdd 100644 Binary files a/Persistence/events.sqlitedb and b/Persistence/events.sqlitedb differ diff --git a/Persistence/schema.ddl b/Persistence/schema.ddl index 4611817..9a41a47 100644 --- a/Persistence/schema.ddl +++ b/Persistence/schema.ddl @@ -3,7 +3,8 @@ date TEXT, timestamp INTEGER, sender TEXT, - intro TEXT + intro TEXT, + entire_message TEXT ); CREATE TABLE calendar_entries( diff --git a/Utils/Date.fs b/Utils/Date.fs index 4f455ed..f013f3c 100644 --- a/Utils/Date.fs +++ b/Utils/Date.fs @@ -2,13 +2,15 @@ open System open EmailParser.Utils.Text +open System.Text.RegularExpressions let secondsSinceEpoch (date: DateTime) = let timeSpanSince1970 = date.Subtract(new DateTime(1970,1,1,0,0,0)) Convert.ToInt64(timeSpanSince1970.TotalSeconds) +let months = [| "jan"; "feb"; "mar"; "apr"; "may"; "jun"; "jul"; "aug"; "sep"; "oct"; "nov"; "dec" |] + let monthIndexFrom (monthString: string) = - let months = [| "jan"; "feb"; "mar"; "apr"; "may"; "jun"; "jul"; "aug"; "sep"; "oct"; "nov"; "dec" |] let thisMonth = monthString.Substring(0,3).ToLower() let index = Array.IndexOf(months, thisMonth) if index = -1 then 1 else (index + 1) @@ -24,15 +26,41 @@ let dateFromMonthDay (monthString: string) (dayString: string) = else proposedDate +let dateFromMonthDayYear (monthString: string) (dayString: string) (yearString: string) = + let monthIndex = monthString.Substring(0,3).ToLower() |> monthIndexFrom + let day = regexReplace @"\D*" "" dayString |> Convert.ToInt32 + let year = regexReplace @"\D*" "" yearString |> Convert.ToInt32 + new DateTime(year, monthIndex, day) + +let isDayOfWeek (dateString: string) = + let possibleWeekday = dateString.Trim().ToLower() + possibleWeekday.StartsWith("mon") || + possibleWeekday.StartsWith("tue") || + possibleWeekday.StartsWith("wed") || + possibleWeekday.StartsWith("thu") || + possibleWeekday.StartsWith("fri") || + possibleWeekday.StartsWith("sat") || + possibleWeekday.StartsWith("sun") + +let isMonth (dateString: string) = + let possibleMonth = dateString.Trim().ToLower() + possibleMonth |> startsWithOneOf months + +let isYear (dateString: string) = + Regex.Match(dateString, @"^\d\d\d\d").Success + +let isTime (possibleTime: string) = + Regex.Match(possibleTime, @"^\d.*[ap]m$").Success + //Assumes that if there is a time, it ends with am/pm let hoursAndMinutesFrom (timeString: string) : option = - if System.String.IsNullOrWhiteSpace(timeString) then - None - else + if isTime timeString then let time = timeString.Substring(0, timeString.Length - 2).Trim() |> regexReplace @"\s+" "" let hoursAndMinutes = time.Split(':') |> Array.map (fun numString -> (Convert.ToDouble numString)) let hours = if timeString.Trim().ToLower().EndsWith("am") then hoursAndMinutes.[0] else hoursAndMinutes.[0] + 12.0 let minutes = if hoursAndMinutes.Length = 2 then hoursAndMinutes.[1] else 0.0 - Some(hours, minutes) \ No newline at end of file + Some(hours, minutes) + else + None \ No newline at end of file diff --git a/Utils/Text.fs b/Utils/Text.fs index 16f243a..353d036 100644 --- a/Utils/Text.fs +++ b/Utils/Text.fs @@ -24,6 +24,8 @@ let splitIntoLines (text: string) = let lineArray = text.Replace("\r\n", "\n").Split( [|'\n'|] ) List.ofArray lineArray +let normalizeSpace (text: string) = text |> regexReplace " +" " " + let asciiSubstitutions = [ ("’", "'") ; ("–", "-") ;