diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 000000000..1ff0c4230
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,63 @@
+###############################################################################
+# Set default behavior to automatically normalize line endings.
+###############################################################################
+* text=auto
+
+###############################################################################
+# Set default behavior for command prompt diff.
+#
+# This is need for earlier builds of msysgit that does not have it on by
+# default for csharp files.
+# Note: This is only used by command line
+###############################################################################
+#*.cs diff=csharp
+
+###############################################################################
+# Set the merge driver for project and solution files
+#
+# Merging from the command prompt will add diff markers to the files if there
+# are conflicts (Merging from VS is not affected by the settings below, in VS
+# the diff markers are never inserted). Diff markers may cause the following
+# file extensions to fail to load in VS. An alternative would be to treat
+# these files as binary and thus will always conflict and require user
+# intervention with every merge. To do so, just uncomment the entries below
+###############################################################################
+#*.sln merge=binary
+#*.csproj merge=binary
+#*.vbproj merge=binary
+#*.vcxproj merge=binary
+#*.vcproj merge=binary
+#*.dbproj merge=binary
+#*.fsproj merge=binary
+#*.lsproj merge=binary
+#*.wixproj merge=binary
+#*.modelproj merge=binary
+#*.sqlproj merge=binary
+#*.wwaproj merge=binary
+
+###############################################################################
+# behavior for image files
+#
+# image files are treated as binary by default.
+###############################################################################
+#*.jpg binary
+#*.png binary
+#*.gif binary
+
+###############################################################################
+# diff behavior for common document formats
+#
+# Convert binary document formats to text before diffing them. This feature
+# is only available from the command line. Turn it on by uncommenting the
+# entries below.
+###############################################################################
+#*.doc diff=astextplain
+#*.DOC diff=astextplain
+#*.docx diff=astextplain
+#*.DOCX diff=astextplain
+#*.dot diff=astextplain
+#*.DOT diff=astextplain
+#*.pdf diff=astextplain
+#*.PDF diff=astextplain
+#*.rtf diff=astextplain
+#*.RTF diff=astextplain
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..581a61332
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,79 @@
+################################################################################
+# This .gitignore file was automatically created by Microsoft(R) Visual Studio.
+################################################################################
+
+
+*.nupkg
+/Source/**/bin/**/*
+/Source/**/obj/**/*
+/Source/**/packages/**/*
+/Source/Actions/*/scripts/**/*
+/Source/Commomn/*/scripts/**/*
+/Source/Actions/Microsoft.Deployment.Actions.Test/StyleCop.Cache
+/Source/Actions/Microsoft.Deployment.Actions.SQL/StyleCop.Cache
+/Source/Actions/Microsoft.Deployment.Actions.OnPremise/StyleCop.Cache
+/Source/Actions/Microsoft.Deployment.Actions.Custom/StyleCop.Cache
+/Source/Actions/Microsoft.Deployment.Actions.Common/StyleCop.Cache
+/Source/Actions/Microsoft.Deployment.Actions.AzureCustom/StyleCop.Cache
+/Source/Actions/Microsoft.Deployment.Actions.ADF/StyleCop.Cache
+*.suo
+**/*/.vs/**/*
+**/*/jspm_packages/**/*
+/Source/Site/Microsoft.Deployment.Site.Web/node_modules
+/Source/Site/Microsoft.Deployment.Site.Web/wwwroot/dist
+*.coverage
+/Source/Actions/.vs/config/applicationhost.config
+/Source/Apps/Microsoft/Released/Microsoft-SCCMTemplate/Resources/SCCM_Sample.pbix
+*.dg
+*.wixpdb
+/Source/Apps/Microsoft/Released/Microsoft-SCCMTemplate/Microsoft-SCCMTemplate.exe
+/Source/Apps/Microsoft/Released/Microsoft-SCCMTemplate/Microsoft-SCCMTemplate.msi
+*.sqlite
+/.vs/slnx.sqlite
+/.vs/slnx.sqlite
+*.vspscc
+/Source/Apps/Microsoft/Released/Microsoft-SapAccountsReceivable/Microsoft-SapAccountsReceivable.exe
+/Private
+/Source/Site/Microsoft.Deployment.Site.Web/typings
+/Source/Site/Microsoft.Deployment.Site.Web/wwwroot/config.js
+/Source/Site/UpgradeLog.htm
+/Source/Site/Microsoft.Deployment.Site.Service/PartnerServiceWSDL/partner.xml
+/Source/Apps/Test/TemplateApps/Microsoft-NewsTemplateTest/v1.txt
+/Source/Apps/Test/TemplateApps/Microsoft-NewsTemplateTest/v2.txt
+/.vs
+/Source/Apps/Microsoft/Released/Microsoft-TwitterTemplate/Service/PowerApp
+/Source/Apps/Microsoft/Released/Simplement-SAP-ARTemplate/Simplement-SAP-ARTemplate.exe
+/Source/Apps/Microsoft/Released/Simplement-SAP-ARTemplate/Simplement-SAP-ARTemplate.msi
+/Source/Apps/Microsoft/Released/Microsoft-SCCM2/Microsoft-SCCM2.exe
+/Source/Apps/Microsoft/Released/Microsoft-SCCM2/Microsoft-SCCM2.msi
+
+/Source/Site/Microsoft.Deployment.Site.Web/global.json
+*.user
+*js.map
+/Source/SiteCommon/Web/**/*.js
+*.pubxml
+/Function/FacebookOAuth/obj/Debug
+/Function/FacebookOAuth/bin/Debug
+/Function/packages
+/Function/FacebookUtillity/obj/Debug
+/Function/FacebookUtilityTest/obj/Debug
+/Function/FacebookUtilityTest/obj/Debug
+/Function/process/bin
+/Function/FacebookUtilityTest/bin/Debug
+/Functions/Code/Facebook/FacebookOAuth/obj/Debug
+/Functions/Code/Facebook/FacebookUtillity/obj/Debug
+/Functions/Code/Facebook/packages
+/Functions/Code/Facebook/process/bin
+/Functions/Code/Facebook/FacebookUtilityTest/obj/Debug
+/Functions/Code/Facebook/FacebookUtilityTest/bin/Debug
+/Function/FacebookUtilityTest/obj/Release
+/Function/FacebookUtillity/obj/Release/CoreCompileInputs.cache
+/Functions/Code/Reddit/Src/RedditAzureFunctions/obj
+/Functions/Code/Reddit/Src/RedditCore/obj
+/Functions/Code/Reddit/Src/RedditAzureFunctions/bin/
+/Functions/Code/Reddit/Src/RedditCore/bin/
+/Functions/Code/Reddit/Test/RedditCoreTest/obj/
+/Functions/Code/Reddit/packages/
+/Functions/Code/Facebook/FacebookUtillity/obj/Release
+/Functions/Code/Facebook/FacebookUtilityTest/obj/Release
+/Functions/Code/Facebook/FacebookOAuth/obj/Release
diff --git a/Functions/Code/Facebook/FacebookETL.sln b/Functions/Code/Facebook/FacebookETL.sln
new file mode 100644
index 000000000..c3b3377b1
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookETL.sln
@@ -0,0 +1,34 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26430.16
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FacebookUtillity", "FacebookUtillity\FacebookUtillity.csproj", "{26D8C540-34DE-4137-8861-A77158B73A49}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FacebookUtilityTest", "FacebookUtilityTest\FacebookUtilityTest.csproj", "{9F47835F-0263-4069-B2AD-03FC4B901B9A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FacebookOAuth", "FacebookOAuth\FacebookOAuth.csproj", "{E87543E4-731A-447E-AA0B-C27499B9A290}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Release|Any CPU = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {26D8C540-34DE-4137-8861-A77158B73A49}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {26D8C540-34DE-4137-8861-A77158B73A49}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {26D8C540-34DE-4137-8861-A77158B73A49}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {26D8C540-34DE-4137-8861-A77158B73A49}.Release|Any CPU.Build.0 = Release|Any CPU
+ {9F47835F-0263-4069-B2AD-03FC4B901B9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {9F47835F-0263-4069-B2AD-03FC4B901B9A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {9F47835F-0263-4069-B2AD-03FC4B901B9A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {9F47835F-0263-4069-B2AD-03FC4B901B9A}.Release|Any CPU.Build.0 = Release|Any CPU
+ {E87543E4-731A-447E-AA0B-C27499B9A290}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {E87543E4-731A-447E-AA0B-C27499B9A290}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {E87543E4-731A-447E-AA0B-C27499B9A290}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {E87543E4-731A-447E-AA0B-C27499B9A290}.Release|Any CPU.Build.0 = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/Functions/Code/Facebook/FacebookOAuth/App.config b/Functions/Code/Facebook/FacebookOAuth/App.config
new file mode 100644
index 000000000..88fa4027b
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookOAuth/App.config
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookOAuth/FacebookOAuth.csproj b/Functions/Code/Facebook/FacebookOAuth/FacebookOAuth.csproj
new file mode 100644
index 000000000..45b343fd6
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookOAuth/FacebookOAuth.csproj
@@ -0,0 +1,65 @@
+
+
+
+
+ Debug
+ AnyCPU
+ {E87543E4-731A-447E-AA0B-C27499B9A290}
+ Exe
+ FacebookOAuth
+ FacebookOAuth
+ v4.5.2
+ 512
+ true
+
+
+ AnyCPU
+ true
+ full
+ false
+ bin\Debug\
+ DEBUG;TRACE
+ prompt
+ 4
+
+
+ AnyCPU
+ pdbonly
+ true
+ bin\Release\
+ TRACE
+ prompt
+ 4
+
+
+
+ ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.16.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll
+
+
+ ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.16.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll
+
+
+ ..\packages\Newtonsoft.Json.6.0.4\lib\net45\Newtonsoft.Json.dll
+ True
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookOAuth/Program.cs b/Functions/Code/Facebook/FacebookOAuth/Program.cs
new file mode 100644
index 000000000..208846af2
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookOAuth/Program.cs
@@ -0,0 +1,87 @@
+using Microsoft.IdentityModel.Clients.ActiveDirectory.Internal;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows.Forms;
+
+namespace FacebookOAuth
+{
+ class Program
+ {
+ public const string clientId = "";
+ public const string clientSecret = "";
+ public const string redirectUri = "https://localhost/";
+ public static string code = string.Empty;
+ public string devCode = "";
+
+ [STAThread]
+ static void Main(string[] args)
+ {
+ code = GetToken(clientId, redirectUri).Result;
+ var shortLivedAccessToken = GetAccessToken($"https://graph.facebook.com/oauth/access_token?client_id={clientId}&client_secret={clientSecret}&redirect_uri={redirectUri}&code={code}").Result;
+ var longLivedAccessToken = GetAccessToken($"https://graph.facebook.com/oauth/access_token?grant_type=fb_exchange_token&client_id={clientId}&client_secret={clientSecret}&fb_exchange_token={shortLivedAccessToken}").Result;
+ var id = GetUserId($"https://graph.facebook.com/v2.10/me?access_token={longLivedAccessToken}").Result;
+ var pageToken = GetAccessToken($"https://graph.facebook.com/v2.10/{id}/accounts?access_token={longLivedAccessToken}").Result;
+ }
+
+ public static async Task GetUserId(string uri)
+ {
+ string requestUri = uri;
+ HttpClient client = new HttpClient();
+ var response = await client.GetAsync(requestUri);
+ string responseObj = await response.Content.ReadAsStringAsync();
+ if (!response.IsSuccessStatusCode)
+ {
+ throw new Exception();
+ }
+
+ string id = JObject.Parse(responseObj)["id"].ToString();
+ return id;
+ }
+
+ public static async Task GetToken(string clientId, string redirectUri)
+ {
+ var url = $"https://www.facebook.com/v2.10/dialog/oauth?client_id={clientId}&redirect_uri={redirectUri}&response_type=code&granted_scopes=manage_pages,publish_pages";
+ var code = string.Empty;
+ WindowsFormsWebAuthenticationDialog form = new WindowsFormsWebAuthenticationDialog(null);
+ form.WebBrowser.Navigated += delegate (object sender, WebBrowserNavigatedEventArgs args)
+ {
+ if (args.Url.ToString().StartsWith("https://localhost"))
+ {
+ string tempcode = args.Url.ToString();
+ tempcode = tempcode.Substring(tempcode.IndexOf("code=") + 5);
+ code = tempcode;
+ form.Close();
+ };
+ };
+ form.WebBrowser.Navigate(url);
+ form.ShowBrowser();
+
+ while (string.IsNullOrEmpty(code))
+ {
+ await Task.Delay(5000);
+ }
+
+ return code;
+ }
+
+ public static async Task GetAccessToken(string uri)
+ {
+ string requestUri = uri;
+ HttpClient client = new HttpClient();
+ var response = await client.GetAsync(requestUri);
+ string responseObj = await response.Content.ReadAsStringAsync();
+ if (!response.IsSuccessStatusCode)
+ {
+ throw new Exception();
+ }
+
+ string accessToken = JObject.Parse(responseObj)["access_token"].ToString();
+ return accessToken;
+ }
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookOAuth/Properties/AssemblyInfo.cs b/Functions/Code/Facebook/FacebookOAuth/Properties/AssemblyInfo.cs
new file mode 100644
index 000000000..afd4a52fe
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookOAuth/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("FacebookOAuth")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("FacebookOAuth")]
+[assembly: AssemblyCopyright("Copyright © 2017")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("e87543e4-731a-447e-aa0b-c27499b9a290")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/Functions/Code/Facebook/FacebookOAuth/packages.config b/Functions/Code/Facebook/FacebookOAuth/packages.config
new file mode 100644
index 000000000..275508acc
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookOAuth/packages.config
@@ -0,0 +1,5 @@
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookUtilityTest/App.config b/Functions/Code/Facebook/FacebookUtilityTest/App.config
new file mode 100644
index 000000000..88fa4027b
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtilityTest/App.config
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookUtilityTest/FacebookUtilityTest.csproj b/Functions/Code/Facebook/FacebookUtilityTest/FacebookUtilityTest.csproj
new file mode 100644
index 000000000..70ae940a4
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtilityTest/FacebookUtilityTest.csproj
@@ -0,0 +1,59 @@
+
+
+
+
+ Debug
+ AnyCPU
+ {9F47835F-0263-4069-B2AD-03FC4B901B9A}
+ Exe
+ FacebookUtilityTest
+ FacebookUtilityTest
+ v4.5.2
+ 512
+ true
+
+
+ AnyCPU
+ true
+ full
+ false
+ bin\Debug\
+ DEBUG;TRACE
+ prompt
+ 4
+
+
+ AnyCPU
+ pdbonly
+ true
+ bin\Release\
+ TRACE
+ prompt
+ 4
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {26D8C540-34DE-4137-8861-A77158B73A49}
+ FacebookUtillity
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookUtilityTest/Program.cs b/Functions/Code/Facebook/FacebookUtilityTest/Program.cs
new file mode 100644
index 000000000..49376dd72
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtilityTest/Program.cs
@@ -0,0 +1,32 @@
+using FacebookETL;
+using FacebookUtillity;
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json.Linq;
+
+namespace FacebookUtilityTest
+{
+ class Program
+ {
+ static void Main(string[] args)
+ {
+ //string sqlConnectionString = "Server = tcp:modb1.database.windows.net,1433; Initial Catalog = fb; Persist Security Info = False; User ID = pbiadmin; Password = Corp123!; MultipleActiveResultSets = False; Encrypt = True; TrustServerCertificate = False; Connection Timeout = 30";
+ //string cognitiveKey = "8f5837f6f20c4374b93d171c490fa58c";
+ //string schema = "[fb]";
+ //string facebookClientId = "421651141539199";
+ //string facebookClientSecret = "511941c6bb0aa06afb250fc5c8628f95";
+
+ //string date = DateTime.Now.AddDays(-2).ToString();
+
+ //var test = MainETL.PopulateAll(sqlConnectionString, schema, cognitiveKey, facebookClientId, facebookClientSecret, date).Result;
+
+ string page = "";
+ string accessToken = "";
+ string sqlConn = "";
+ string schema = "";
+ string until = DateTime.UtcNow.AddDays(-2).ToString();
+
+ var test = PageAnalyticsETL.PopulateMeasures(page, accessToken, sqlConn, schema, until).Result;
+ }
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtilityTest/Properties/AssemblyInfo.cs b/Functions/Code/Facebook/FacebookUtilityTest/Properties/AssemblyInfo.cs
new file mode 100644
index 000000000..636d05f8c
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtilityTest/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("FacebookUtilityTest")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("FacebookUtilityTest")]
+[assembly: AssemblyCopyright("Copyright © 2017")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("9f47835f-0263-4069-b2ad-03fc4b901b9a")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/Functions/Code/Facebook/FacebookUtillity/CognitiveUtility.cs b/Functions/Code/Facebook/FacebookUtillity/CognitiveUtility.cs
new file mode 100644
index 000000000..693b81916
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/CognitiveUtility.cs
@@ -0,0 +1,135 @@
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FacebookUtillity
+{
+ public class CognitiveUtility
+ {
+ public static void PopulateDictionary(Dictionary dictionary, DataTable table)
+ {
+ foreach (DataRow row in table.Rows)
+ {
+ if (!dictionary.ContainsKey(row["Id"].ToString()) && !string.IsNullOrEmpty(row["message"]?.ToString()))
+ {
+ dictionary.Add(row["Id"].ToString(), row["message"].ToString());
+ }
+ }
+ }
+
+
+ public static List GetPayloads(Dictionary dictionary)
+ {
+ JObject objTemp = new JObject();
+ List objDocArray = new List();
+ var objArray = new JArray();
+
+ for (int i = 0; i < dictionary.Count; i++)
+ {
+ var item = dictionary.Keys.ElementAt(i);
+
+ if (i % 1000 == 0 && i != 0)
+ {
+ objTemp.Add("documents", objArray);
+ objArray = new JArray();
+ objDocArray.Add(objTemp);
+ objTemp = new JObject();
+ }
+
+ if (!(string.IsNullOrEmpty(item) && string.IsNullOrEmpty(dictionary[item]) && string.IsNullOrWhiteSpace(dictionary[item])))
+ {
+ JObject doc = new JObject();
+ doc.Add("id", item);
+ doc.Add("text", dictionary[item]);
+ objArray.Add(doc);
+ }
+ }
+
+ if (objArray.Count > 0)
+ {
+ objTemp.Add("documents", objArray);
+ objDocArray.Add(objTemp);
+ }
+
+ return objDocArray;
+ }
+
+ public static async Task GetSentimentAsync(List payloads, DataTable sentiment, string cognitiveKey)
+ {
+ foreach (var payload in payloads)
+ {
+ HttpClient client = new HttpClient();
+ client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", cognitiveKey);
+ HttpContent content = new StringContent(payload.ToString(), System.Text.Encoding.UTF8, "application/json");
+ var response = await client.PostAsync("https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/sentiment", content);
+ string result = await response.Content.ReadAsStringAsync();
+ if (response.IsSuccessStatusCode)
+ {
+ JObject responseObj = JObject.Parse(result);
+
+ foreach (var doc in responseObj["documents"])
+ {
+ string[] split = doc["id"].ToString().Split('_');
+ DataRow row = sentiment.NewRow();
+ row["Id"] = doc["id"];
+ row["Sentiment"] = double.Parse(doc["score"].ToString());
+ sentiment.Rows.Add(row);
+ }
+ }
+ }
+ }
+
+ public static async Task GetKeyPhraseAsync(List payloads, DataTable keyPhrase, string cognitiveKey)
+ {
+ foreach (var payload in payloads)
+ {
+ HttpClient client = new HttpClient();
+ client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", cognitiveKey);
+ HttpContent content = new StringContent(payload.ToString(), System.Text.Encoding.UTF8, "application/json");
+ var response = await client.PostAsync("https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/keyPhrases", content);
+ if (response.IsSuccessStatusCode)
+ {
+ string result = await response.Content.ReadAsStringAsync();
+ JObject responseObj = JObject.Parse(result);
+ foreach (var doc in responseObj["documents"])
+ {
+ foreach (var keyword in doc["keyPhrases"])
+ {
+ if (string.IsNullOrEmpty(keyword?.ToString()))
+ {
+ continue;
+ }
+
+ DataRow row = keyPhrase.NewRow();
+ row["Id"] = doc["id"];
+ int maxLength = Math.Min(keyword.ToString().Length, 100);
+ row["KeyPhrase"] = keyword.ToString().ToLowerInvariant().Substring(0, maxLength);
+ keyPhrase.Rows.Add(row);
+ }
+ }
+ }
+ }
+ }
+
+ public static void GetHashTags(DataTable postsOrComments, DataTable hashTagsDataTable)
+ {
+ foreach (DataRow post in postsOrComments.Rows)
+ {
+ var hashTags = Utility.ExtractHashTag(post["message"]?.ToString());
+ foreach (var hashTag in hashTags)
+ {
+ DataRow row = hashTagsDataTable.NewRow();
+ row["Id"] = post["Id"];
+ int maxLength = Math.Min(hashTag.ToString().Length, 100);
+ row["HashTags"] = hashTag.ToString().ToLowerInvariant().Substring(0, maxLength);
+ hashTagsDataTable.Rows.Add(row);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookUtillity/DataTableUtility.cs b/Functions/Code/Facebook/FacebookUtillity/DataTableUtility.cs
new file mode 100644
index 000000000..64deada68
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/DataTableUtility.cs
@@ -0,0 +1,333 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FacebookUtillity
+{
+ public class DataTableUtility
+ {
+ #region Standard FB Tables
+ public static DataTable GetCommentsDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("Created Date", typeof(DateTime));
+ table.Columns.Add("Message");
+ table.Columns.Add("From Id");
+ table.Columns.Add("From Name");
+ table.Columns.Add("Post Id");
+ table.Columns.Add("Page");
+ table.Columns.Add("PageDisplayName");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetReactionsDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("Reaction Type");
+ table.Columns.Add("Count", typeof(long));
+ return table;
+ }
+
+ public static DataTable GetPostsDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("Created Date", typeof(DateTime));
+ table.Columns.Add("Message");
+ table.Columns.Add("From Id");
+ table.Columns.Add("From Name");
+ table.Columns.Add("Media");
+ table.Columns.Add("Page");
+ table.Columns.Add("PageDisplayName");
+ table.Columns.Add("PageId");
+ table.Columns.Add("Total Comments", typeof(double));
+ return table;
+ }
+
+
+ public static DataTable GetSentimentDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("Sentiment", typeof(float));
+ return table;
+ }
+
+ public static DataTable GetKeyPhraseDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("KeyPhrase");
+ return table;
+ }
+
+ public static DataTable GetHashTagDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("HashTags");
+ return table;
+ }
+
+ public static DataTable GetErrorDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Date", typeof(DateTime));
+ table.Columns.Add("Error");
+ table.Columns.Add("Posts");
+ return table;
+ }
+ #endregion
+
+ #region Page Analytics Tables
+ public static DataTable GetPagePostStoriesAndPeopleTalkingAboutThisTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GePageImpressionsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageEngagementTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageReactionsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageClicksDataTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageUserDemographicsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageContentTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageViewsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageVideoViewsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPagePostsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPagePostImpressionsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPagePostEngagementTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("EndTime", typeof(DateTime));
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPagePostReactionsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPageVideoPostsTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Name");
+ table.Columns.Add("Entry Name");
+ table.Columns.Add("Value");
+ table.Columns.Add("Period");
+ table.Columns.Add("Title");
+ table.Columns.Add("Description");
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ return table;
+ }
+
+ public static DataTable GetPostsInfoTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ table.Columns.Add("Message");
+ table.Columns.Add("Created Time", typeof(DateTime));
+ table.Columns.Add("Updated Time", typeof(DateTime));
+ table.Columns.Add("Icon");
+ table.Columns.Add("Story");
+ table.Columns.Add("Link");
+ table.Columns.Add("Status Type");
+ table.Columns.Add("Is Hidden");
+ table.Columns.Add("Is Published");
+ table.Columns.Add("Name");
+ table.Columns.Add("Object");
+ table.Columns.Add("Permalink URL");
+ table.Columns.Add("Picture");
+ table.Columns.Add("Source");
+ table.Columns.Add("Shares");
+ table.Columns.Add("Type");
+ return table;
+ }
+
+ public static DataTable GetPostsToTable()
+ {
+ DataTable table = new DataTable();
+ table.Columns.Add("Id");
+ table.Columns.Add("PageId");
+ table.Columns.Add("Created Time", typeof(DateTime));
+ table.Columns.Add("Updated Time", typeof(DateTime));
+ table.Columns.Add("To Id");
+ table.Columns.Add("To Name");
+ return table;
+ }
+ #endregion
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtillity/DateUtility.cs b/Functions/Code/Facebook/FacebookUtillity/DateUtility.cs
new file mode 100644
index 000000000..9a5d10b87
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/DateUtility.cs
@@ -0,0 +1,51 @@
+using System;
+
+namespace FacebookETL
+{
+ public class DateUtility
+ {
+ public static string GetDate(string daysAsString)
+ {
+ // Set name to query string or body data
+ int days = int.Parse(daysAsString);
+
+ if (days > 0)
+ {
+ days = days * -1;
+ }
+
+ string dateToReturn = DateTime.Now.AddDays(days).ToString();
+ return dateToReturn;
+ }
+
+ public static string GetUnixFromDate(string date)
+ {
+ // Get request body
+ DateTime dateTime = DateTime.Parse(date);
+ DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, 0);
+ TimeSpan span = (dateTime.ToUniversalTime() - epoch);
+ string timeStamp = Math.Floor(span.TotalSeconds).ToString();
+ return timeStamp;
+ }
+
+ public static string GetDateTimeRelativeFromNow(int days)
+ {
+ if (days > 0)
+ {
+ days = days * -1;
+ }
+
+ return DateTime.Now.AddDays(days).ToString();
+ }
+
+ public static string GetDateTimeRelativeFromNow(string date, int days)
+ {
+ if (days > 0)
+ {
+ days = days * -1;
+ }
+
+ return DateTime.Parse(date).AddDays(days).ToString();
+ }
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtillity/FacebookPageAnalyticsMetricGroups.cs b/Functions/Code/Facebook/FacebookUtillity/FacebookPageAnalyticsMetricGroups.cs
new file mode 100644
index 000000000..93a4e932e
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/FacebookPageAnalyticsMetricGroups.cs
@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FacebookUtillity
+{
+ public static class FacebookPageAnalyticsMetricGroups
+ {
+ public const string PagePostStoriesAndPeopleTalkingAboutThis = "pagePostStoriesAndPeopleTalkingAboutThis";
+ public const string PageImpressions = "pageImpressions";
+ public const string PageEngagement = "pageEngagement";
+ public const string PageReactions = "pageReactions";
+ public const string PageCtaClicks = "pageCtaClicks";
+ public const string PageUserDemographics = "pageUserDemographics";
+ public const string PageContent = "pageContent";
+ public const string PageViews = "pageViews";
+ public const string PageVideoViews = "pageVideoViews";
+ public const string PagePosts = "pagePosts";
+ public const string PagePostImpressions = "pagePostImpressions";
+ public const string PagePostEngagement = "pagePostEngagement";
+ public const string PagePostReactions = "pagePostReactions";
+ public const string PageVideoPosts = "pageVideoPosts";
+ public const string PagePostIds = "pagePostIds";
+
+ public const string PagePostStoriesAndPeopleTalkingAboutThisMetrics = "page_content_activity_by_action_type_unique,page_content_activity_by_age_gender_unique,page_content_activity_by_city_unique,page_content_activity_by_country_unique,page_content_activity_by_locale_unique,page_content_activity,page_content_activity_by_action_type,post_activity,post_activity_unique,post_activity_by_action_type,post_activity_by_action_type_unique,";
+ public const string PageImpressionsMetrics = "page_impressions,page_impressions_unique,page_impressions_paid,page_impressions_paid_unique,page_impressions_organic,page_impressions_organic_unique,page_impressions_viral,page_impressions_viral_unique,page_impressions_by_story_type,page_impressions_by_story_type_unique,page_impressions_by_city_unique,page_impressions_by_country_unique,page_impressions_by_locale_unique,page_impressions_by_age_gender_unique,page_impressions_frequency_distribution,page_impressions_viral_frequency_distribution";
+ public const string PageEngagementMetrics = "page_engaged_users,page_post_engagements,page_consumptions,page_consumptions_unique,page_consumptions_by_consumption_type,page_consumptions_by_consumption_type_unique,page_places_checkin_total,page_places_checkin_total_unique,page_places_checkin_mobile,page_places_checkin_mobile_unique,page_places_checkins_by_age_gender,page_places_checkins_by_locale,page_places_checkins_by_country,page_negative_feedback,page_negative_feedback_unique,page_negative_feedback_by_type,page_negative_feedback_by_type_unique,page_positive_feedback_by_type,page_positive_feedback_by_type_unique,page_fans_online,page_fans_online_per_day,page_fan_adds_by_paid_non_paid_unique";
+ public const string PageReactionsMetrics = "page_actions_post_reactions_total";
+ public const string PageCtaClicksMetrics = "page_total_actions,page_cta_clicks_logged_in_total,page_cta_clicks_logged_in_unique,page_cta_clicks_by_site_logged_in_unique,page_cta_clicks_by_age_gender_logged_in_unique,page_cta_clicks_logged_in_by_country_unique,page_cta_clicks_logged_in_by_city_unique,page_call_phone_clicks_logged_in_unique,page_call_phone_clicks_by_age_gender_logged_in_unique,page_call_phone_clicks_logged_in_by_country_unique,page_call_phone_clicks_logged_in_by_city_unique,page_call_phone_clicks_by_site_logged_in_unique,page_get_directions_clicks_logged_in_unique,page_get_directions_clicks_by_age_gender_logged_in_unique,page_get_directions_clicks_logged_in_by_country_unique,page_get_directions_clicks_logged_in_by_city_unique,page_get_directions_clicks_by_site_logged_in_unique,page_website_clicks_logged_in_unique,page_website_clicks_by_age_gender_logged_in_unique,page_website_clicks_logged_in_by_country_unique,page_website_clicks_logged_in_by_city_unique,page_website_clicks_by_site_logged_in_unique";
+ public const string PageUserDemographicsMetrics = "page_fans,page_fans_locale,page_fans_city,page_fans_country,page_fans_gender_age,page_fan_adds,page_fan_adds_unique,page_fans_by_like_source,page_fans_by_like_source_unique,page_fan_removes,page_fan_removes_unique,page_fans_by_unlike_source_unique";
+ public const string PageContentMetrics = "page_consumptions_by_consumption_type,page_tab_views_login_top_unique,page_tab_views_login_top,page_tab_views_logout_top";
+ public const string PageViewsMetrics = "page_views_total,page_views_logout,page_views_logged_in_total,page_views_logged_in_unique,page_views_external_referrals,page_views_by_profile_tab_total,page_views_by_profile_tab_logged_in_unique,page_views_by_internal_referer_logged_in_unique,page_views_by_site_logged_in_unique,page_views_by_age_gender_logged_in_unique,page_views,page_views_unique,page_views_login,page_views_login_unique,page_visits_logged_in_by_referers_unique";
+ public const string PageVideoViewsMetrics = "page_video_views,page_video_views_paid,page_video_views_organic,page_video_views_by_paid_non_paid,page_video_views_autoplayed,page_video_views_click_to_play,page_video_views_unique,page_video_repeat_views,page_video_views_10s,page_video_views_10s_paid,page_video_views_10s_organic,page_video_views_10s_autoplayed,page_video_views_10s_click_to_play,page_video_views_10s_unique,page_video_views_10s_repeat,page_video_view_time";
+ public const string PagePostsMetrics = "page_posts_impressions,page_posts_impressions_unique,page_posts_impressions_paid,page_posts_impressions_paid_unique,page_posts_impressions_organic,page_posts_impressions_organic_unique,page_posts_impressions_viral,page_posts_impressions_viral_unique,page_posts_impressions_frequency_distribution";
+ public const string PagePostImpressionsMetrics = "post_impressions,post_impressions_unique,post_impressions_paid,post_impressions_paid_unique,post_impressions_fan,post_impressions_fan_unique,post_impressions_fan_paid,post_impressions_fan_paid_unique,post_impressions_organic,post_impressions_organic_unique,post_impressions_viral,post_impressions_viral_unique,post_impressions_nonviral,post_impressions_nonviral_unique,post_impressions_by_story_type,post_impressions_by_story_type_unique";
+ public const string PagePostEngagementMetrics = "post_negative_feedback,post_negative_feedback_unique,post_negative_feedback_by_type,post_negative_feedback_by_type_unique,post_engaged_fan";
+ public const string PagePostReactionsMetrics = "post_reactions_by_type_total,post_engaged_users";
+ public const string PageVideoPostsMetrics = "post_video_avg_time_watched,post_video_complete_views_organic,post_video_complete_views_organic_unique,post_video_complete_views_paid,post_video_complete_views_paid_unique,post_video_retention_graph,post_video_retention_graph_clicked_to_play,post_video_retention_graph_autoplayed,post_video_views_organic,post_video_views_organic_unique,post_video_views_paid,post_video_views_paid_unique,post_video_length,post_video_views,post_video_views_unique,post_video_views_autoplayed,post_video_views_clicked_to_play,post_video_views_10s,post_video_views_10s_unique,post_video_views_10s_autoplayed,post_video_views_10s_clicked_to_play,post_video_views_10s_organic,post_video_views_10s_paid,post_video_views_10s_sound_on,post_video_views_sound_on,post_video_view_time,post_video_view_time_organic,post_video_view_time_by_age_bucket_and_gender,post_video_view_time_by_region_id,post_video_views_by_distribution_type,post_video_view_time_by_distribution_type,post_video_view_time_by_country_id";
+ public const string PagePostIdsMetrics = "id";
+
+ public static Dictionary Metrics = new Dictionary()
+ {
+ {PagePostStoriesAndPeopleTalkingAboutThis, PagePostStoriesAndPeopleTalkingAboutThisMetrics},
+ {PageImpressions,PageImpressionsMetrics },
+ {PageEngagement,PageEngagementMetrics },
+ {PageReactions,PageReactionsMetrics },
+ {PageCtaClicks,PageCtaClicksMetrics },
+ {PageUserDemographics,PageUserDemographicsMetrics },
+ {PageContent,PageContentMetrics },
+ {PageViews,PageViewsMetrics },
+ {PageVideoViews,PageVideoViewsMetrics },
+ {PagePosts,PagePostsMetrics },
+ {PagePostImpressions,PagePostImpressionsMetrics },
+ {PagePostEngagement,PagePostEngagementMetrics },
+ {PagePostReactions,PagePostReactionsMetrics },
+ {PageVideoPosts,PageVideoPostsMetrics },
+ {PagePostIds,PagePostIdsMetrics }
+ };
+
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtillity/FacebookUtility.cs b/Functions/Code/Facebook/FacebookUtillity/FacebookUtility.cs
new file mode 100644
index 000000000..b790b9623
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/FacebookUtility.cs
@@ -0,0 +1,226 @@
+using FacebookETL;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Web;
+using System.Threading;
+
+namespace FacebookUtillity
+{
+ public class FacebookUtility
+ {
+ const int daysToGoBack = 0;
+
+ public static async Task GetPage(string page, string accessToken)
+ {
+ string requestUri = $"https://graph.facebook.com/{page}?access_token={accessToken}";
+ HttpClient client = new HttpClient();
+ var response = await client.GetAsync(requestUri);
+ string responseObj = await response.Content.ReadAsStringAsync();
+ return JObject.Parse(responseObj);
+ }
+
+ public static async Task> GetPostsAsync(string page, string untilDateTime, string accessToken)
+ {
+ List posts = new List();
+ JObject post = null;
+ string until = DateUtility.GetUnixFromDate(untilDateTime);
+ string since = DateUtility.GetUnixFromDate(DateUtility.GetDateTimeRelativeFromNow(untilDateTime, -1));
+ string requestUri = GetRequestUrlForPage(page, accessToken, until, since);
+ do
+ {
+ HttpClient client = new HttpClient();
+ var response = await client.GetAsync(requestUri);
+ string responseObj = await response.Content.ReadAsStringAsync();
+ if (!response.IsSuccessStatusCode)
+ {
+ throw new Exception();
+ }
+
+
+ post = JObject.Parse(responseObj);
+ posts.Add(post);
+
+ if (post?["paging"] != null && post["paging"]?["cursors"] != null && post["paging"]["cursors"]?["after"] != null)
+ {
+ string after = post["paging"]["cursors"]["after"].ToString();
+ requestUri = GetRequestUrlForPage(page, accessToken, until, since, after);
+
+ }
+ else if (post?["paging"] != null && post["paging"]?["next"] != null)
+ {
+
+ requestUri = post["paging"]?["next"].ToString();
+ }
+ }
+ while (post != null && post?["paging"] != null);
+
+ return posts;
+
+ }
+
+ public static async Task GetAccessTokenAsync(string clientId, string clientSecret)
+ {
+ string requestUri = $"https://graph.facebook.com/oauth/access_token?grant_type=client_credentials&client_id={clientId}&client_secret={clientSecret}";
+ HttpClient client = new HttpClient();
+ var response = await client.GetAsync(requestUri);
+ string responseObj = await response.Content.ReadAsStringAsync();
+ if (!response.IsSuccessStatusCode)
+ {
+ throw new Exception();
+ }
+
+ string accessToken = JObject.Parse(responseObj)["access_token"].ToString();
+ return accessToken;
+ }
+
+ public static string GetRequestUrlForPage(string page, string accessToken, string until, string since, string after = "")
+ {
+ Dictionary param = new Dictionary();
+ param.Add("access_token", accessToken);
+ param.Add("fields", "message,updated_time,created_time,comments.limit(100).order(chronological).summary(true),from,picture" +
+ ",reactions.type(LOVE).limit(0).summary(total_count).as(reactions_love)" +
+ ",reactions.type(WOW).limit(0).summary(total_count).as(reactions_wow)" +
+ ",reactions.type(HAHA).limit(0).summary(total_count).as(reactions_haha)" +
+ ",reactions.type(SAD).limit(0).summary(total_count).as(reactions_sad)" +
+ ",reactions.type(ANGRY).limit(0).summary(total_count).as(reactions_angry)" +
+ ",reactions.type(LIKE).limit(0).summary(total_count).as(reactions_like)");
+
+ param.Add("until", until);
+ param.Add("since", since);
+ if (!string.IsNullOrEmpty(after))
+ {
+ param.Add("after", after);
+ }
+
+ return $"https://graph.facebook.com/v2.9/{page}/feed?" + GetQueryParameters(param);
+ }
+
+ #region Page Analytics
+ public static async Task> GetPageMetricAnalytics(string page, string untilDateTime, string accessToken, string metricsGroup)
+ {
+ List posts = new List();
+ JObject post = null;
+ string until = DateUtility.GetUnixFromDate(untilDateTime);
+ string since = DateUtility.GetUnixFromDate(DateUtility.GetDateTimeRelativeFromNow(untilDateTime, daysToGoBack));
+ string requestUri = metricsGroup == FacebookPageAnalyticsMetricGroups.PagePostIds ?
+ GetPagePostIds(page, accessToken, untilDateTime, since) :
+ GetPageAnalyticsRequestUrl(page, accessToken, until, since, metricsGroup);
+
+ do
+ {
+ using (var client = new HttpClient())
+ {
+ var response = await client.GetAsync(requestUri);
+ string responseObj = await response.Content.ReadAsStringAsync();
+
+ if (!response.IsSuccessStatusCode)
+ {
+ Thread.Sleep(new TimeSpan(0, 0, 3));
+ response = await client.GetAsync(requestUri);
+ responseObj = await response.Content.ReadAsStringAsync();
+ if (!response.IsSuccessStatusCode)
+ {
+ throw new Exception(responseObj);
+ }
+ }
+
+ post = JObject.Parse(responseObj);
+
+ if (!posts.Contains(post) && (post?["data"] as JArray).Count > 0)
+ {
+ posts.Add(post);
+ }
+
+ if (post?["paging"] != null && post["paging"]?["cursors"] != null && post["paging"]["cursors"]?["after"] != null)
+ {
+ string after = post["paging"]["cursors"]["after"].ToString();
+ requestUri = GetRequestUrlForPage(page, accessToken, until, since, after);
+
+ }
+ else if (post?["paging"] != null && post["paging"]?["next"] != null)
+ {
+ requestUri = post["paging"]?["next"].ToString();
+ }
+ }
+
+ if (post?["data"] != null && (post?["data"] as JArray).Count > 0)
+ {
+ var endTime = post?["data"]?[0]?["values"]?[0]?["end_time"];
+
+ if (endTime != null)
+ {
+ var et = DateTime.Parse(endTime.ToString());
+ var untilTime = DateTime.Parse(untilDateTime);
+ if (untilTime.Subtract(et).TotalDays < 1)
+ {
+ break;
+ }
+ }
+ }
+
+ if (post?["data"] != null &&
+ (post["data"] as JArray).Count > 0 &&
+ post?["data"]?[0]?["period"] != null &&
+ post?["data"]?[0]?["period"].ToString().ToLower() == "lifetime")
+ {
+ break;
+ }
+ }
+ while (post != null && post?["paging"] != null && post["paging"]?["next"] != null);
+
+ return posts;
+ }
+
+ private static string GetPagePostIds(string page, string accessToken, string untilDateTime, string after = "")
+ {
+ string until = DateUtility.GetUnixFromDate(untilDateTime);
+ string since = DateUtility.GetUnixFromDate(DateUtility.GetDateTimeRelativeFromNow(untilDateTime, daysToGoBack -2));
+ Dictionary param = new Dictionary();
+ param.Add("access_token", accessToken);
+ param.Add("fields", "id,message,updated_time,created_time,icon,link,name,object_id,permalink_url,picture,source,shares,to,type,story,status_type,is_hidden,is_published");
+ param.Add("until", until);
+
+ param.Add("since", since);
+
+ return $"https://graph.facebook.com/v2.10/{page}/posts?" + GetQueryParameters(param);
+ }
+
+ private static string GetPageAnalyticsRequestUrl(string page, string accessToken, string until, string since, string metricsGroup, string after = "")
+ {
+ Dictionary param = new Dictionary();
+ param.Add("access_token", accessToken);
+ param.Add("metric", FacebookPageAnalyticsMetricGroups.Metrics[metricsGroup]);
+ param.Add("until", until);
+ param.Add("since", since);
+
+ if (!string.IsNullOrEmpty(after))
+ {
+ param.Add("after", after);
+ }
+
+ return $"https://graph.facebook.com/v2.10/{page}/insights?" + GetQueryParameters(param);
+ }
+
+ #endregion
+
+ private static string GetQueryParameters(Dictionary queryParams)
+ {
+ string str = "";
+
+ foreach (var queryParam in queryParams)
+ {
+ if (!string.IsNullOrEmpty(str))
+ {
+ str += "&";
+ }
+
+ str += queryParam.Key + "=" + HttpUtility.UrlEncode(queryParam.Value);
+ }
+
+ return str;
+ }
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookUtillity/FacebookUtillity.csproj b/Functions/Code/Facebook/FacebookUtillity/FacebookUtillity.csproj
new file mode 100644
index 000000000..01f5a1a67
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/FacebookUtillity.csproj
@@ -0,0 +1,62 @@
+
+
+
+
+ Debug
+ AnyCPU
+ {26D8C540-34DE-4137-8861-A77158B73A49}
+ Library
+ Properties
+ FacebookUtillity
+ FacebookUtillity
+ v4.5.2
+ 512
+
+
+ true
+ full
+ false
+ ..\process\bin\
+ DEBUG;TRACE
+ prompt
+ 4
+
+
+ pdbonly
+ true
+ bin\Release\
+ TRACE
+ prompt
+ 4
+
+
+
+ ..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Facebook/FacebookUtillity/MainETL.cs b/Functions/Code/Facebook/FacebookUtillity/MainETL.cs
new file mode 100644
index 000000000..be7acaf24
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/MainETL.cs
@@ -0,0 +1,180 @@
+using FacebookETL;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FacebookUtillity
+{
+ public class MainETL
+ {
+ public static async Task PopulateAll(string sqlConn, string schema, string cognitiveKey, string client, string secret, string date)
+ {
+ string token = await FacebookUtility.GetAccessTokenAsync(client, secret);
+ string[] pages = SqlUtility.GetPages(sqlConn, schema);
+
+ List posts = new List();
+
+ foreach (var pageToSearch in pages)
+ {
+ string page = pageToSearch.Replace(" ", "");
+ try
+ {
+ var pageObj = await FacebookUtility.GetPage(page, token);
+
+ // Get Facebook Posts
+ posts = await FacebookUtility.GetPostsAsync(page, date, token);
+ // Get All Data Tables
+ var commentsDataTable = DataTableUtility.GetCommentsDataTable();
+ var hashTagDataTable = DataTableUtility.GetHashTagDataTable();
+ var keyPhraseDataTable = DataTableUtility.GetKeyPhraseDataTable();
+ var postDataTable = DataTableUtility.GetPostsDataTable();
+ var reactionsDataTable = DataTableUtility.GetReactionsDataTable();
+ var sentimentDataTable = DataTableUtility.GetSentimentDataTable();
+
+ PopulatePostCommentsAndReactions(postDataTable, commentsDataTable, reactionsDataTable, posts, page, pageObj);
+
+ // Populate Sentiment
+ Dictionary items = new Dictionary();
+ CognitiveUtility.PopulateDictionary(items, postDataTable);
+ CognitiveUtility.PopulateDictionary(items, commentsDataTable);
+ var payloads = CognitiveUtility.GetPayloads(items);
+
+ await CognitiveUtility.GetSentimentAsync(payloads, sentimentDataTable, cognitiveKey);
+ await CognitiveUtility.GetKeyPhraseAsync(payloads, keyPhraseDataTable, cognitiveKey);
+ CognitiveUtility.GetHashTags(postDataTable, hashTagDataTable);
+ CognitiveUtility.GetHashTags(commentsDataTable, hashTagDataTable);
+
+ // Bulk Insert
+ SqlUtility.BulkInsert(sqlConn, postDataTable, schema + "." + "StagingPosts");
+ SqlUtility.BulkInsert(sqlConn, sentimentDataTable, schema + "." + "StagingSentiment");
+ SqlUtility.BulkInsert(sqlConn, commentsDataTable, schema + "." + "StagingComments");
+ SqlUtility.BulkInsert(sqlConn, keyPhraseDataTable, schema + "." + "StagingKeyPhrase");
+ SqlUtility.BulkInsert(sqlConn, reactionsDataTable, schema + "." + "StagingReactions");
+ SqlUtility.BulkInsert(sqlConn, hashTagDataTable, schema + "." + "StagingHashTags");
+
+ // Debugging
+ var errorDataTable = DataTableUtility.GetErrorDataTable();
+ DataRow errorRow = errorDataTable.NewRow();
+ errorRow["Date"] = date;
+ errorRow["Error"] = "";
+ errorRow["Posts"] = page + ":" + JToken.FromObject(posts).ToString();
+ errorDataTable.Rows.Add(errorRow);
+ SqlUtility.BulkInsert(sqlConn, errorDataTable, schema + "." + "StagingError");
+ }
+ catch (Exception e)
+ {
+ var errorDataTable = DataTableUtility.GetErrorDataTable();
+ DataRow errorRow = errorDataTable.NewRow();
+ errorRow["Date"] = date;
+ errorRow["Error"] = e.ToString();
+ errorRow["Posts"] = page + ":" + JToken.FromObject(posts).ToString();
+ errorDataTable.Rows.Add(errorRow);
+ SqlUtility.BulkInsert(sqlConn, errorDataTable, schema + "." + "StagingError");
+ throw;
+ }
+ }
+ return true;
+ }
+
+ public static void PopulatePostCommentsAndReactions(DataTable postsDataTable, DataTable commentsDataTable,
+ DataTable reactionsDataTable, List posts, string page, JObject pageObj)
+ {
+ foreach (var postPayload in posts)
+ {
+ foreach (var post in postPayload["data"])
+ {
+ DataRow postRow = postsDataTable.NewRow();
+ postRow["Id"] = post["id"];
+ postRow["Created Date"] = post["created_time"];
+ postRow["Message"] = post["message"];
+ postRow["From Id"] = post["from"]?["id"];
+ int maxLength = 0;
+ if (post["from"]?["name"] != null)
+ {
+ maxLength = Math.Min((post["from"]["name"].ToString().Length), 100);
+ }
+ postRow["From Name"] = post["from"]?["name"].ToString().Substring(0, maxLength);
+ postRow["Media"] = post["picture"];
+ postRow["Page"] = page;
+ postRow["PageId"] = pageObj["id"].ToString();
+ postRow["PageDisplayName"] = pageObj["name"];
+
+ if (post["comments"]["data"].Count() == 100)
+ {
+ postRow["Total Comments"] = Utility.ConvertToLong(post["comments"]["summary"]["total_count"]); ;
+ }
+ else
+ {
+ postRow["Total Comments"] = post["comments"]["data"].Count();
+ }
+
+ postsDataTable.Rows.Add(postRow);
+
+
+ foreach (var comment in post["comments"]["data"])
+ {
+
+ DataRow commentRow = commentsDataTable.NewRow();
+ commentRow["Id"] = comment["id"];
+ commentRow["Created Date"] = comment["created_time"];
+ commentRow["Message"] = comment["message"];
+ commentRow["From Id"] = comment["from"]?["id"];
+ if (comment["from"]?["name"] != null)
+ {
+ maxLength = Math.Min(comment["from"]["name"].ToString().Length, 100);
+ }
+ commentRow["From Name"] = comment["from"]?["name"].ToString().Substring(0, maxLength);
+ commentRow["Post Id"] = post["id"];
+ commentRow["Page"] = page;
+ commentRow["PageId"] = pageObj["id"].ToString();
+ commentRow["PageDisplayName"] = pageObj["name"];
+ commentsDataTable.Rows.Add(commentRow);
+ }
+
+
+ DataRow likeRow = reactionsDataTable.NewRow();
+ likeRow["Id"] = post["id"];
+ likeRow["Reaction Type"] = "Like";
+ likeRow["Count"] = Utility.ConvertToLong(post["reactions_like"]["summary"]["total_count"]);
+
+ DataRow hahaRow = reactionsDataTable.NewRow();
+ hahaRow["Id"] = post["id"];
+ hahaRow["Reaction Type"] = "Haha";
+ hahaRow["Count"] = Utility.ConvertToLong(post["reactions_haha"]["summary"]["total_count"]);
+
+ DataRow sadRow = reactionsDataTable.NewRow();
+ sadRow["Id"] = post["id"];
+ sadRow["Reaction Type"] = "Sad";
+ sadRow["Count"] = Utility.ConvertToLong(post["reactions_sad"]["summary"]["total_count"]);
+
+ DataRow loveRow = reactionsDataTable.NewRow();
+ loveRow["Id"] = post["id"];
+ loveRow["Reaction Type"] = "Love";
+ loveRow["Count"] = Utility.ConvertToLong(post["reactions_love"]["summary"]["total_count"]);
+
+ DataRow angryRow = reactionsDataTable.NewRow();
+ angryRow["Id"] = post["id"];
+ angryRow["Reaction Type"] = "Angry";
+ angryRow["Count"] = Utility.ConvertToLong(post["reactions_angry"]["summary"]["total_count"]);
+
+ DataRow wowRow = reactionsDataTable.NewRow();
+ wowRow["Id"] = post["id"];
+ wowRow["Reaction Type"] = "Wow";
+ wowRow["Count"] = Utility.ConvertToLong(post["reactions_wow"]["summary"]["total_count"]);
+
+ reactionsDataTable.Rows.Add(likeRow);
+ reactionsDataTable.Rows.Add(hahaRow);
+ reactionsDataTable.Rows.Add(sadRow);
+ reactionsDataTable.Rows.Add(loveRow);
+ reactionsDataTable.Rows.Add(angryRow);
+ reactionsDataTable.Rows.Add(wowRow);
+
+ }
+ }
+ }
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtillity/PageAnalyticsETL.cs b/Functions/Code/Facebook/FacebookUtillity/PageAnalyticsETL.cs
new file mode 100644
index 000000000..7a437c86d
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/PageAnalyticsETL.cs
@@ -0,0 +1,301 @@
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Diagnostics;
+using FacebookETL;
+
+namespace FacebookUtillity
+{
+ public class PageAnalyticsETL
+ {
+ public static async Task PopulateMeasures(string pageId, string pageAccessToken, string sqlConnection, string sqlSchema, string getDataUntil)
+ {
+ string page = pageId;
+ string accessToken = pageAccessToken;
+ string sqlConn = sqlConnection;
+ string schema = sqlSchema;
+ string until = getDataUntil;
+ try
+ {
+ var pageContentTable = DataTableUtility.GetPageContentTable();
+ var pageEngagementTable = DataTableUtility.GetPageEngagementTable();
+ var pageImpressionsTable = DataTableUtility.GePageImpressionsTable();
+ var pagePostsTable = DataTableUtility.GetPagePostsTable();
+ var pagePostEngagement = DataTableUtility.GetPagePostEngagementTable();
+ var pagePostImpressions = DataTableUtility.GetPagePostImpressionsTable();
+ var pagePostReactionsTable = DataTableUtility.GetPagePostReactionsTable();
+ var pagePostStoriesAndPeopleTalkingAboutThisTable = DataTableUtility.GetPagePostStoriesAndPeopleTalkingAboutThisTable();
+ var pageReactionsTable = DataTableUtility.GetPageReactionsTable();
+ var pageUserDemographicsTable = DataTableUtility.GetPageUserDemographicsTable();
+ var pageVideoPosts = DataTableUtility.GetPageVideoPostsTable();
+ var pageVideoViews = DataTableUtility.GetPageVideoViewsTable();
+ var pageViewsTable = DataTableUtility.GetPageViewsTable();
+ var clicksTable = DataTableUtility.GetPageClicksDataTable();
+ var postsInfoTable = DataTableUtility.GetPostsInfoTable();
+ var postsToTable = DataTableUtility.GetPostsToTable();
+
+ var content = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageContent);
+ PopulateNestedValues(pageContentTable, content, page);
+
+ var engagement = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageEngagement);
+ PopulateNestedValues(pageEngagementTable, engagement, page);
+
+ var impressions = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageImpressions);
+ PopulateNestedValues(pageImpressionsTable, impressions, page);
+
+ var pagePosts = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PagePosts);
+ PopulateNestedValues(pagePostsTable, pagePosts, page);
+
+ var pagePostsEngagement = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PagePostEngagement);
+ PopulateNestedValues(pagePostEngagement, pagePostsEngagement, page);
+
+ var pagePostsReactions = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PagePostReactions);
+ PopulateNestedValues(pagePostReactionsTable, pagePostsReactions, page);
+
+ var pageReactions = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageReactions);
+ PopulateNestedValues(pageReactionsTable, pageReactions, page);
+
+ var pagePostStories = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PagePostStoriesAndPeopleTalkingAboutThis);
+ PopulateNestedValues(pagePostStoriesAndPeopleTalkingAboutThisTable, pagePostStories, page);
+
+ var pageUserDemographics = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageUserDemographics);
+ PopulateNestedValues(pageUserDemographicsTable, pageUserDemographics, page);
+
+ var pageVideoViewsObj = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageVideoViews);
+ PopulateNestedValues(pageVideoViews, pageVideoViewsObj, page);
+
+ var pageViews = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageViews);
+ PopulateNestedValues(pageViewsTable, pageViews, page);
+
+ var clicks = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PageCtaClicks);
+ PopulateNestedValues(clicksTable, clicks, page);
+
+ var pagePostIds = await FacebookUtility.GetPageMetricAnalytics(page, until, accessToken, FacebookPageAnalyticsMetricGroups.PagePostIds);
+ PopulatePostsInfo(postsInfoTable, pagePostIds, page);
+ PopulatePostsTo(postsToTable, pagePostIds, page);
+
+ List pagePostReactions = new List();
+ List pageVideoPostsObj = new List();
+ List pagePostsImpressions = new List();
+ if (pagePostIds != null)
+ {
+ foreach (var entry in pagePostIds)
+ {
+ if (entry?["data"] != null)
+ {
+ foreach (var obj in entry["data"])
+ {
+ pagePostsImpressions.AddRange(await FacebookUtility.GetPageMetricAnalytics(obj["id"].ToString(), until, accessToken, FacebookPageAnalyticsMetricGroups.PagePostImpressions));
+ pageVideoPostsObj.AddRange(await FacebookUtility.GetPageMetricAnalytics(obj["id"].ToString(), until, accessToken, FacebookPageAnalyticsMetricGroups.PageVideoPosts));
+ pagePostReactions.AddRange(await FacebookUtility.GetPageMetricAnalytics(obj["id"].ToString(), until, accessToken, FacebookPageAnalyticsMetricGroups.PagePostReactions));
+ }
+ }
+ }
+ }
+ PopulateNestedValues(pagePostReactionsTable, pagePostReactions, page);
+ PopulateNestedValues(pageVideoPosts, pageVideoPostsObj, page);
+ PopulateNestedValues(pagePostImpressions, pagePostsImpressions, page);
+
+ SqlUtility.BulkInsert(sqlConn, pageContentTable, schema + "." + "STAGING_PageContent");
+ SqlUtility.BulkInsert(sqlConn, pageEngagementTable, schema + "." + "STAGING_PageEngagement");
+ SqlUtility.BulkInsert(sqlConn, pageImpressionsTable, schema + "." + "STAGING_PageImpressions");
+ SqlUtility.BulkInsert(sqlConn, pagePostsTable, schema + "." + "STAGING_PagePost");
+ SqlUtility.BulkInsert(sqlConn, pagePostEngagement, schema + "." + "STAGING_PagePostEngagement");
+ SqlUtility.BulkInsert(sqlConn, pagePostImpressions, schema + "." + "STAGING_PagePostImpressions");
+ SqlUtility.BulkInsert(sqlConn, pagePostReactionsTable, schema + "." + "STAGING_PagePostReactions");
+ SqlUtility.BulkInsert(sqlConn, pagePostStoriesAndPeopleTalkingAboutThisTable, schema + "." + "STAGING_PagePostStoriesAndPeopleTalkingAboutThis");
+ SqlUtility.BulkInsert(sqlConn, pageReactionsTable, schema + "." + "STAGING_PageReactions");
+ SqlUtility.BulkInsert(sqlConn, pageUserDemographicsTable, schema + "." + "STAGING_PageUserDemographics");
+ SqlUtility.BulkInsert(sqlConn, pageVideoPosts, schema + "." + "STAGING_PageVideoPosts");
+ SqlUtility.BulkInsert(sqlConn, pageVideoViews, schema + "." + "STAGING_PageVideoViews");
+ SqlUtility.BulkInsert(sqlConn, pageViewsTable, schema + "." + "STAGING_PageViews");
+ SqlUtility.BulkInsert(sqlConn, clicksTable, schema + "." + "STAGING_Clicks");
+ SqlUtility.BulkInsert(sqlConn, postsInfoTable, schema + "." + "STAGING_PagePostsInfo");
+ SqlUtility.BulkInsert(sqlConn, postsToTable, schema + "." + "STAGING_PagePostsTo");
+
+ }
+ catch (Exception e)
+ {
+ var errorDataTable = DataTableUtility.GetErrorDataTable();
+ DataRow errorRow = errorDataTable.NewRow();
+ errorRow["Date"] = getDataUntil == string.Empty ? DateTime.UtcNow.ToString("o") : getDataUntil;
+ errorRow["Error"] = e.ToString();
+ errorDataTable.Rows.Add(errorRow);
+ SqlUtility.BulkInsert(sqlConn, errorDataTable, schema + "." + "Error");
+ throw;
+ }
+ return true;
+ }
+
+ public static void PopulateNestedValues(DataTable table, List objects, string pageId)
+ {
+ foreach (var obj in objects)
+ {
+ foreach (var entry in obj["data"])
+ {
+ foreach (var val in entry["values"])
+ {
+ if (val?["value"] != null && val["value"].Children().Count() >= 1)
+ {
+ Extract(table, pageId, entry, val);
+ }
+ else
+ {
+ DataRow row = table.NewRow();
+ if (table.Columns.Contains("EndTime")) { row["EndTime"] = val["end_time"]; }
+ row["Name"] = entry["name"];
+ row["Value"] = (val["value"] != null && string.IsNullOrEmpty(val["value"].ToString())) ||
+ (val["value"] != null && val["value"].ToString() == "{}") ?
+ DBNull.Value :
+ (object)val["value"];
+ Debug.WriteLine(row["Value"]);
+ row["Period"] = entry["period"];
+ row["Title"] = entry["title"];
+ row["Description"] = entry["description"];
+ row["Id"] = entry["id"];
+ row["PageId"] = pageId;
+ table.Rows.Add(row);
+ }
+ }
+ }
+ }
+ }
+
+ private static void Extract(DataTable table, string pageId, JToken entry, JToken val)
+ {
+ foreach (var child in val["value"])
+ {
+ if (child.GetType() != typeof(JProperty) &&
+ child?["value"] != null &&
+ child["value"].Children().Count() >= 1)
+ {
+ Extract(table, pageId, entry, child);
+ }
+ else
+ {
+ var att = child as JProperty;
+
+ if (att.Value.Children().Count() >= 1)
+ {
+ ExtractChildren(table, pageId, entry, val, att);
+ }
+ else
+ {
+ DataRow row = table.NewRow();
+ if (table.Columns.Contains("EndTime")) { row["EndTime"] = val["end_time"]; }
+ row["Name"] = entry["name"];
+ row["Entry Name"] = att.Name;
+ row["Value"] = (att.Value != null && string.IsNullOrEmpty(att.Value.ToString())) ||
+ (att.Value != null && att.Value.ToString() == "{}") ?
+ DBNull.Value :
+ (object)att.Value;
+ Debug.WriteLine(row["Value"]);
+ row["Period"] = entry["period"];
+ row["Title"] = entry["title"];
+ row["Description"] = entry["description"];
+ row["Id"] = entry["id"];
+ row["PageId"] = pageId;
+ table.Rows.Add(row);
+ }
+ }
+ }
+ }
+
+ private static void ExtractChildren(DataTable table, string pageId, JToken entry, JToken val, JProperty att)
+ {
+ if (att.Value.Children().Count() >= 1)
+ {
+ foreach (var c in att.Value.Children())
+ {
+ ExtractChildren(table, pageId, entry, val, c as JProperty);
+ }
+ }
+ else
+ {
+ DataRow row = table.NewRow();
+ if (table.Columns.Contains("EndTime")) { row["EndTime"] = val["end_time"]; }
+ row["Name"] = entry["name"];
+ //Only merge names for age / gender
+ if (entry["name"].ToString().ToLower().Contains("page_cta_clicks_by_age_gender_logged_in_unique"))
+ {
+ row["Entry Name"] = att.Name + " " + (att.Parent.Parent as JProperty).Name;
+ }
+ else
+ {
+ row["Entry Name"] = att.Name;
+ }
+ row["Value"] = (att.Value != null && string.IsNullOrEmpty(att.Value.ToString())) ||
+ (att.Value != null && att.Value.ToString() == "{}") ?
+ DBNull.Value :
+ (object)att.Value;
+ Debug.WriteLine(row["Value"]);
+ row["Period"] = entry["period"];
+ row["Title"] = entry["title"];
+ row["Description"] = entry["description"];
+ row["Id"] = entry["id"];
+ row["PageId"] = pageId;
+ table.Rows.Add(row);
+ }
+ }
+
+ public static void PopulatePostsInfo(DataTable table, List objects, string pageId)
+ {
+ foreach (var obj in objects)
+ {
+ foreach (var val in obj["data"])
+ {
+ DataRow row = table.NewRow();
+ row["Id"] = val["id"];
+ row["PageId"] = pageId;
+ row["Message"] = val["message"];
+ row["Created Time"] = val["created_time"];
+ row["Updated Time"] = val["updated_time"];
+ row["Icon"] = val["icon"];
+ row["Link"] = val["link"];
+ row["Name"] = val["name"];
+ row["Object"] = val["object_id"];
+ row["Permalink URL"] = val["permalink_url"];
+ row["Picture"] = val["picture"];
+ row["Source"] = val["source"];
+ row["Shares"] = val["shares"]?["count"] == null ? DBNull.Value : (object)val["shares"]?["count"];
+ row["Type"] = val["type"];
+ row["Status Type"] = val["status_type"];
+ row["Is Hidden"] = val["is_hidden"];
+ row["Is Published"] = val["is_published"];
+ row["Story"] = val["story"];
+ table.Rows.Add(row);
+ }
+ }
+ }
+
+ public static void PopulatePostsTo(DataTable table, List objects, string pageId)
+ {
+ foreach (var obj in objects)
+ {
+ foreach (var val in obj["data"])
+ {
+ if (val?["to"] != null &&
+ (val["to"]["data"] as JArray).Count > 0)
+ {
+ foreach (var t in val["to"]["data"])
+ {
+ DataRow row = table.NewRow();
+ row["Id"] = val["id"];
+ row["PageId"] = pageId;
+ row["Created Time"] = val["created_time"];
+ row["Updated Time"] = val["updated_time"];
+ row["To Id"] = t["id"];
+ row["To Name"] = t["name"];
+ table.Rows.Add(row);
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtillity/Properties/AssemblyInfo.cs b/Functions/Code/Facebook/FacebookUtillity/Properties/AssemblyInfo.cs
new file mode 100644
index 000000000..493711b25
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("FacebookUtillity")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("FacebookUtillity")]
+[assembly: AssemblyCopyright("Copyright © 2017")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("26d8c540-34de-4137-8861-a77158b73a49")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/Functions/Code/Facebook/FacebookUtillity/SqlUtility.cs b/Functions/Code/Facebook/FacebookUtillity/SqlUtility.cs
new file mode 100644
index 000000000..1c0bb098f
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/SqlUtility.cs
@@ -0,0 +1,72 @@
+using System;
+using System.Data;
+using System.Data.SqlClient;
+using System.Text.RegularExpressions;
+
+namespace FacebookETL
+{
+ public class SqlUtility
+ {
+ public static string SanitizeSchemaName(string value)
+ {
+ if (string.IsNullOrWhiteSpace(value))
+ return string.Empty;
+
+ Regex invalidCharacters = new Regex("[^a-zA-Z_0-9]");
+ return invalidCharacters.Replace(value, string.Empty);
+ }
+
+ public static void RunStoredProc(string sqlConnectionString, string storedProc)
+ {
+ using (SqlConnection conn = new SqlConnection(sqlConnectionString))
+ {
+ conn.Open();
+ using (SqlCommand command = new SqlCommand(storedProc, conn) { CommandType = CommandType.StoredProcedure, CommandTimeout = 180 })
+ {
+ command.ExecuteScalar();
+ }
+ }
+ }
+
+ public static void BulkInsert(string connString, DataTable table, string tableName)
+ {
+ try
+ {
+ using (SqlBulkCopy bulk = new SqlBulkCopy(connString))
+ {
+ bulk.BatchSize = 5000;
+ bulk.DestinationTableName = tableName;
+ bulk.WriteToServer(table);
+ bulk.Close();
+ }
+ }
+ catch
+ {
+ throw new Exception("overflow during batch insert in table " + tableName);
+ }
+ }
+
+ public static string[] GetPages(string sqlConnectionString, string schema)
+ {
+ string pagesCommaSeparated = null;
+ string SQL_PAGES_TO_FOLLOW = $"SELECT [value] FROM {SanitizeSchemaName(schema)}.[configuration] WHERE [configuration_group] = 'SolutionTemplate' AND [configuration_subgroup] = 'ETL' AND [name] = 'PagesToFollow'";
+
+ using (SqlConnection conn = new SqlConnection(sqlConnectionString))
+ {
+ conn.Open();
+ using (SqlCommand command = new SqlCommand(SQL_PAGES_TO_FOLLOW, conn))
+ {
+ SqlDataReader dr = command.ExecuteReader();
+ if (dr.Read())
+ {
+ if (dr[0] != null && dr[0] != DBNull.Value)
+ pagesCommaSeparated = dr[0].ToString();
+ }
+ dr.Close();
+ }
+ }
+
+ return pagesCommaSeparated != null ? pagesCommaSeparated.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries) : new string[] { };
+ }
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtillity/Utility.cs b/Functions/Code/Facebook/FacebookUtillity/Utility.cs
new file mode 100644
index 000000000..32f05ef88
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/Utility.cs
@@ -0,0 +1,54 @@
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+namespace FacebookUtillity
+{
+ public class Utility
+ {
+ public static long ConvertToLong(JToken value)
+ {
+ string strValue = value?.ToString();
+ if (string.IsNullOrEmpty(strValue))
+ {
+ return 0;
+ }
+ else
+ {
+ return long.Parse(strValue);
+ }
+ }
+
+ public static long ConvertToLong(string value)
+ {
+ string strValue = value?.ToString();
+ if (string.IsNullOrEmpty(strValue))
+ {
+ return 0;
+ }
+ else
+ {
+ return long.Parse(strValue);
+ }
+ }
+
+ public static List ExtractHashTag(string message)
+ {
+ List hashTags = new List();
+ if (!string.IsNullOrEmpty(message))
+ {
+ var matches = Regex.Matches(message, "(\\#\\w+) ");
+ foreach (Match match in matches)
+ {
+ hashTags.Add(match.Value);
+ }
+ }
+
+ return hashTags;
+ }
+ }
+}
diff --git a/Functions/Code/Facebook/FacebookUtillity/packages.config b/Functions/Code/Facebook/FacebookUtillity/packages.config
new file mode 100644
index 000000000..810e559c0
--- /dev/null
+++ b/Functions/Code/Facebook/FacebookUtillity/packages.config
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Reddit/PowerBI/RedditAdvancedAnalytics.pbit b/Functions/Code/Reddit/PowerBI/RedditAdvancedAnalytics.pbit
new file mode 100644
index 000000000..f6db57904
Binary files /dev/null and b/Functions/Code/Reddit/PowerBI/RedditAdvancedAnalytics.pbit differ
diff --git a/Functions/Code/Reddit/PowerBI/RedditOverview.pbit b/Functions/Code/Reddit/PowerBI/RedditOverview.pbit
new file mode 100644
index 000000000..7e41847db
Binary files /dev/null and b/Functions/Code/Reddit/PowerBI/RedditOverview.pbit differ
diff --git a/Functions/Code/Reddit/PowerBI/RedditTargetingActivation.pbit b/Functions/Code/Reddit/PowerBI/RedditTargetingActivation.pbit
new file mode 100644
index 000000000..81b41e11e
Binary files /dev/null and b/Functions/Code/Reddit/PowerBI/RedditTargetingActivation.pbit differ
diff --git a/Functions/Code/Reddit/README.md b/Functions/Code/Reddit/README.md
new file mode 100644
index 000000000..8d87f3b7e
--- /dev/null
+++ b/Functions/Code/Reddit/README.md
@@ -0,0 +1,20 @@
+#Introduction
+Pipeline that ingests Reddit data for use in PowerBI.
+
+## Important Note
+This folder structure has been exported from another git repository. It is meant to be a reference for anyone who wants to see how this pipeline works, make changes, publish their own, and update their own Reddit ingest and ML pipeline. Synchronization of internal repository to BusinessPlatformApps Github repository is manual.
+
+## Projects
+ - RedditAzureFunctions
+ - RedditCore
+ - RedditDatabase - *Note*: This should be a 1:1 clone of the BusinessPlatformApps/Source/Apps/Microsoft/Release/Microsoft-RedditTemplate/Database folder. Deltas should be treated (and reported!) as bugs. This is included solely to simplify the already manual sync process.
+ - RedditCoreTest
+
+## Description
+RedditCore contains the bulk of the processing logic. It exists primarily so that our unit test project, RedditCoreTest, can use and access it. A project of type Azure Function did not seem to give us any unit test capability. The RedditAzureFunction is all about the set up and configuration of the function app that powers this Reddit ingest and analytics pipeline. A local.settings.json file exists in the root of the RedditAzureFunctions folder and contains all of the properties necessary to run this (to run locally, please see: [Code and Test Azure Functions Locally](https://docs.microsoft.com/en-us/azure/azure-functions/functions-run-local) ).
+
+You will need to create an instance of the Brand and Campaign Management for Reddit from Appsource to have a SocialGist key provisioned. You can use this key for development, but the key is limited in uses per day and could be disabled if it runs afoul of SocialGist usage rules. Please contact SocialGist for if you wish to do something more strenuous.
+
+If you are merely testing the initial ingest portion, you can comment out the FunctionName annotation on ScheduledAzureML and AzureMLJobChecker. You can then run completely locally using an embedded SQL database.
+
+However, if you want to test the AzureML portion as well, you will need to create a SQL Server and Database in Azure. You will also need to provision the AzureML portion, which can be found in BusinessPlatformApps/Source/Apps/Microsoft/Release/Microsoft-RedditTemplate/Service/AzureML. This file will need to be modified by hand to point to your own database, storage account connection strings, TwitterModel.ilearner file, etc. After this AzureML Web Service is created, you can use the address and web service key in the local.settings.json. You will also need to change the SqlConnection in ConnectionStrings to point to your Azure SQL Database. You can still run the FunctionApp locally, but the AzureML processing will be done in Azure. Making changes to AzureML are quite problematic, though if you are interested you could probably reverse engineer the current web service into an AzureML experiment in your own workspace, modify it as needed, and build your own web service. This is a complex process and well beyond the scope of this README.
\ No newline at end of file
diff --git a/Functions/Code/Reddit/RedditTemplate.sln b/Functions/Code/Reddit/RedditTemplate.sln
new file mode 100644
index 000000000..e8e52199a
--- /dev/null
+++ b/Functions/Code/Reddit/RedditTemplate.sln
@@ -0,0 +1,45 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26730.3
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RedditCoreTest", "Test\RedditCoreTest\RedditCoreTest.csproj", "{AFD1103A-6060-427B-A530-D1FD3377524F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RedditCore", "Src\RedditCore\RedditCore.csproj", "{887D2B7C-34E2-4585-8DD7-FD622349712E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RedditAzureFunctions", "Src\RedditAzureFunctions\RedditAzureFunctions.csproj", "{034E74CE-0D1E-4FCD-BAC6-9C18B791F4AB}"
+EndProject
+Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "RedditDatabase", "Src\RedditDatabase\RedditDatabase.sqlproj", "{9A7DD0D7-D825-42BA-8F2A-70CB90BBD321}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Release|Any CPU = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {AFD1103A-6060-427B-A530-D1FD3377524F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {AFD1103A-6060-427B-A530-D1FD3377524F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {AFD1103A-6060-427B-A530-D1FD3377524F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {AFD1103A-6060-427B-A530-D1FD3377524F}.Release|Any CPU.Build.0 = Release|Any CPU
+ {887D2B7C-34E2-4585-8DD7-FD622349712E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {887D2B7C-34E2-4585-8DD7-FD622349712E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {887D2B7C-34E2-4585-8DD7-FD622349712E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {887D2B7C-34E2-4585-8DD7-FD622349712E}.Release|Any CPU.Build.0 = Release|Any CPU
+ {034E74CE-0D1E-4FCD-BAC6-9C18B791F4AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {034E74CE-0D1E-4FCD-BAC6-9C18B791F4AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {034E74CE-0D1E-4FCD-BAC6-9C18B791F4AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {034E74CE-0D1E-4FCD-BAC6-9C18B791F4AB}.Release|Any CPU.Build.0 = Release|Any CPU
+ {9A7DD0D7-D825-42BA-8F2A-70CB90BBD321}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {9A7DD0D7-D825-42BA-8F2A-70CB90BBD321}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {9A7DD0D7-D825-42BA-8F2A-70CB90BBD321}.Debug|Any CPU.Deploy.0 = Debug|Any CPU
+ {9A7DD0D7-D825-42BA-8F2A-70CB90BBD321}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {9A7DD0D7-D825-42BA-8F2A-70CB90BBD321}.Release|Any CPU.Build.0 = Release|Any CPU
+ {9A7DD0D7-D825-42BA-8F2A-70CB90BBD321}.Release|Any CPU.Deploy.0 = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+ GlobalSection(ExtensibilityGlobals) = postSolution
+ SolutionGuid = {8637E0CA-BFAB-4906-8D2D-8C3E6A56434B}
+ EndGlobalSection
+EndGlobal
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/AzureMLJobChecker.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/AzureMLJobChecker.cs
new file mode 100644
index 000000000..ed03ddda7
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/AzureMLJobChecker.cs
@@ -0,0 +1,76 @@
+using System;
+using System.Configuration;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Azure.WebJobs.Host;
+using Ninject;
+using RedditCore;
+using RedditAzureFunctions.Logging;
+using RedditCore.AzureML;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Queue;
+
+namespace RedditAzureFunctions
+{
+ public static class AzureMLJobChecker
+ {
+ ///
+ /// Perform application initialization on startup
+ ///
+ static AzureMLJobChecker()
+ {
+ new Bootstrap().Init();
+ }
+
+ private const string FunctionName = nameof(AzureMLJobChecker);
+
+ [FunctionName(FunctionName)]
+ [Singleton(Mode = SingletonMode.Function)]
+ public static async Task Run(
+ [QueueTrigger(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] string jobId,
+ ExecutionContext executionContext,
+ TraceWriter logger
+ )
+ {
+ logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
+ IConfiguration webConfiguration = new WebConfiguration(executionContext);
+ var log = new FunctionLog(logger, executionContext.InvocationId);
+
+ var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;
+
+ using (var kernel = new KernelFactory().GetKernel(
+ log,
+ webConfiguration,
+ objectLogger
+ ))
+ {
+
+ var processor = kernel.Get();
+ var result = await processor.CheckAzureMLAndPostProcess(jobId);
+
+ if (!result.LastJobStatus.IsTerminalState())
+ {
+ var queue = CreateQueue();
+
+ // Job is not finished. Put it back on the queue to try again.
+ var message = new CloudQueueMessage(jobId);
+ queue.AddMessage(message, null, webConfiguration.AzureMlRetryTimeDelay);
+ }
+ }
+
+ logger.Info($"{FunctionName} completed at {DateTime.Now}");
+ }
+
+ private static CloudQueue CreateQueue()
+ {
+ var connectionString = ConfigurationManager.AppSettings[QueueConstants.QueueConnectionStringName];
+ var queueName = ConfigurationManager.AppSettings[QueueConstants.AzureMLJobQueueNameProperty];
+ var storageAccount = CloudStorageAccount.Parse(connectionString);
+
+ var queueClient = storageAccount.CreateCloudQueueClient();
+ var queue = queueClient.GetQueueReference(queueName);
+
+ return queue;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/Bootstrap.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/Bootstrap.cs
new file mode 100644
index 000000000..14caab2e4
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/Bootstrap.cs
@@ -0,0 +1,16 @@
+using System.Data.Entity;
+using RedditCore.DataModel;
+
+namespace RedditAzureFunctions
+{
+ public class Bootstrap
+ {
+ public void Init()
+ {
+ // Normally you do this initialization by using the DbConfigurationTypeAttribute
+ // on the DbContext object. Could not get the attribute method to work
+ // so fell back on this static initialization
+ DbConfiguration.SetConfiguration(new RedditDatabaseConfiguration());
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/ExecuteRedditSearch.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/ExecuteRedditSearch.cs
new file mode 100644
index 000000000..eeb95eba0
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/ExecuteRedditSearch.cs
@@ -0,0 +1,139 @@
+using System;
+using System.Collections.Generic;
+using System.Configuration;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.ApplicationInsights.DataContracts;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Azure.WebJobs.Host;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Queue;
+using Newtonsoft.Json;
+using Ninject;
+using RedditCore;
+using RedditAzureFunctions.Logging;
+using RedditCore.SocialGist;
+using RedditCore.DataModel;
+using RedditCore.Telemetry;
+using ExecutionContext = Microsoft.Azure.WebJobs.ExecutionContext;
+
+namespace RedditAzureFunctions
+{
+ public static class ExecuteRedditSearch
+ {
+ private const string FunctionName = nameof(ExecuteRedditSearch);
+
+ ///
+ /// Perform application initialization on startup
+ ///
+ static ExecuteRedditSearch()
+ {
+ new Bootstrap().Init();
+ }
+
+ [FunctionName(FunctionName)]
+ public static void Run(
+ [QueueTrigger(QueueConstants.RedditSearchQueueName, Connection = QueueConstants.QueueConnectionStringName)] string processMessage,
+ ExecutionContext executionContext,
+ TraceWriter logger
+ )
+ {
+ logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
+ IConfiguration webConfiguration = new WebConfiguration(executionContext);
+ var log = new FunctionLog(logger, executionContext.InvocationId);
+
+ var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;
+ using (var kernel = new KernelFactory().GetKernel(
+ log,
+ webConfiguration,
+ objectLogger
+ ))
+ {
+
+ var socialGist = kernel.Get();
+ var telemetry = kernel.Get();
+ socialGist.ResultLimitPerPage = webConfiguration.ResultLimitPerPage;
+ socialGist.MaximumResultsPerSearch = webConfiguration.MaximumResultsPerSearch;
+
+ SortedSet threadMatches = null;
+ using (var sgQueryTelemetry =
+ telemetry.StartTrackDependency("Execute Search", null, "SocialGistPostSearch"))
+ {
+ threadMatches = socialGist.MatchesForQuery(
+ webConfiguration.QueryTerms,
+ webConfiguration.QuerySortOrder,
+ null
+ ).Result;
+
+ sgQueryTelemetry.IsSuccess = true;
+ }
+
+ logger.Info(
+ $"Returned [{threadMatches.Count}] posts from search terms [{webConfiguration.QueryTerms}]");
+
+ using (var queueCollectorTelemetry =
+ telemetry.StartTrackDependency("Enqueue Results", null, "SocialGistPostSearch"))
+ {
+
+ var timeDelay = webConfiguration.SearchToThreadTimeDelay;
+ var queue = CreateQueue();
+
+ queue.CreateIfNotExists();
+
+ QueueRequestOptions queueRequestOptions = new QueueRequestOptions()
+ {
+ MaximumExecutionTime = TimeSpan.FromMinutes(1)
+ };
+
+ var q = new HashSet();
+
+ // Write to the queue. By default this will use will utilize however many threads the underlying scheduler provides.
+ // See https://docs.microsoft.com/en-us/dotnet/api/system.threading.tasks.paralleloptions.maxdegreeofparallelism?view=netframework-4.7.1#System_Threading_Tasks_ParallelOptions_MaxDegreeOfParallelism
+ Parallel.ForEach(
+ threadMatches,
+ CreateQueue,
+ (item, loopState, innerQueue) =>
+ {
+ q.Add(Thread.CurrentThread.ManagedThreadId);
+ var messageContent = JsonConvert.SerializeObject(item);
+ var message = new CloudQueueMessage(messageContent);
+ innerQueue.AddMessage(message, options: queueRequestOptions,
+ initialVisibilityDelay: timeDelay);
+ return innerQueue;
+ },
+ (finalResult) => { }
+ );
+
+ queueCollectorTelemetry.Properties.Add("Total Number of Threads Used", q.Count.ToString());
+ queueCollectorTelemetry.IsSuccess = true;
+ }
+
+ var metric = new MetricTelemetry()
+ {
+ Name = "Unique posts returned by search",
+ Sum = threadMatches.Count,
+ Timestamp = DateTime.Now,
+ Properties =
+ {
+ {"QueryTerms", webConfiguration.QueryTerms },
+ }
+ };
+ telemetry.TrackMetric(metric);
+ }
+
+ logger.Info($"{FunctionName} completed at {DateTime.Now}");
+ }
+
+ private static CloudQueue CreateQueue()
+ {
+ var connectionString = ConfigurationManager.AppSettings[QueueConstants.QueueConnectionStringName];
+ var queueName = ConfigurationManager.AppSettings[QueueConstants.RedditPostQueueNameProperty];
+ var storageAccount = CloudStorageAccount.Parse(connectionString);
+
+ var queueClient = storageAccount.CreateCloudQueueClient();
+ var queue = queueClient.GetQueueReference(queueName);
+
+ return queue;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/Logging/BlobObjectLogger.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/Logging/BlobObjectLogger.cs
new file mode 100644
index 000000000..4e16c10bf
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/Logging/BlobObjectLogger.cs
@@ -0,0 +1,71 @@
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Auth;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Newtonsoft.Json;
+using RedditCore;
+using RedditCore.Logging;
+using System;
+using System.IO;
+using System.Text;
+
+namespace RedditAzureFunctions.Logging
+{
+ internal class BlobObjectLogger : IObjectLogger
+ {
+ private readonly IConfiguration configuration;
+ private readonly ILog logger;
+
+ internal BlobObjectLogger(IConfiguration configuration, ILog logger)
+ {
+ this.configuration = configuration;
+ this.logger = logger;
+ }
+
+ public void Log(object data, string source, string message = null)
+ {
+ try
+ {
+ string dataString = (data != null) ? data as string : "null";
+ if (dataString == null)
+ {
+ dataString = JsonConvert.SerializeObject(data);
+ }
+
+ var sasUri = new Uri(this.configuration.ObjectLogSASUrl);
+ var accountName = sasUri.Host.TrimEndString(".blob.core.windows.net");
+ StorageCredentials creds = new StorageCredentials(sasUri.Query);
+ CloudStorageAccount strAcc = new CloudStorageAccount(creds, accountName, endpointSuffix: null, useHttps: true);
+ CloudBlobClient blobClient = strAcc.CreateCloudBlobClient();
+
+ //Setup our container we are going to use and create it.
+ CloudBlobContainer container = blobClient.GetContainerReference(configuration.ObjectLogBlobContainer);
+ container.CreateIfNotExistsAsync();
+
+ // Build my typical log file name.
+ DateTime date = DateTime.Now;
+
+ // This creates a reference to the append blob we are going to use.
+ CloudAppendBlob appBlob = container.GetAppendBlobReference(
+ $"{date.ToString("yyyy-MM")}/{date.ToString("dd")}/{date.ToString("HH")}-{source}.log");
+
+ // Now we are going to check if todays file exists and if it doesn't we create it.
+ if (!appBlob.Exists())
+ {
+ appBlob.CreateOrReplace();
+ }
+
+ // Add the entry to our log.
+ var logMessage = $"{date.ToString("o")}|{message}|{dataString}{Environment.NewLine}";
+ using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(logMessage)))
+ {
+ // Append a block. AppendText is not safe across multiple threads & servers so use AppendBlock instead.
+ appBlob.AppendBlock(ms);
+ }
+ }
+ catch (Exception ex)
+ {
+ this.logger.Error("Error logging data to the permanent store", ex);
+ }
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/Logging/FunctionLog.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/Logging/FunctionLog.cs
new file mode 100644
index 000000000..544ed303e
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/Logging/FunctionLog.cs
@@ -0,0 +1,43 @@
+using System;
+using Microsoft.Azure.WebJobs.Host;
+using RedditCore.Logging;
+
+namespace RedditAzureFunctions.Logging
+{
+ internal class FunctionLog : ILog
+ {
+ private readonly TraceWriter traceWriter;
+ private readonly Guid invocationId;
+
+ internal FunctionLog(TraceWriter traceWriter, Guid invocationId)
+ {
+ this.traceWriter = traceWriter;
+ this.invocationId = invocationId;
+ }
+
+ public void Error(string message, Exception ex = null, string source = null)
+ {
+ traceWriter.Error(GetMessage(message), ex, source);
+ }
+
+ public void Info(string message, string source = null)
+ {
+ traceWriter.Info(GetMessage(message), source);
+ }
+
+ public void Verbose(string message, string source = null)
+ {
+ traceWriter.Verbose(GetMessage(message), source);
+ }
+
+ public void Warning(string message, string source = null)
+ {
+ traceWriter.Warning(GetMessage(message), source);
+ }
+
+ private string GetMessage(string message)
+ {
+ return $"[Invocation {invocationId}] {message}";
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/ProcessPost.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/ProcessPost.cs
new file mode 100644
index 000000000..e1626fdc2
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/ProcessPost.cs
@@ -0,0 +1,51 @@
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Azure.WebJobs.Host;
+using Ninject;
+using RedditAzureFunctions.Logging;
+using RedditCore;
+using RedditCore.DataModel;
+using RedditCore.SocialGist;
+using System;
+
+namespace RedditAzureFunctions
+{
+ public static class ProcessPost
+ {
+ private const string FunctionName = nameof(ProcessPost);
+
+ ///
+ /// Perform application initialization on startup
+ ///
+ static ProcessPost()
+ {
+ new Bootstrap().Init();
+ }
+
+ [FunctionName(FunctionName)]
+ public static async Task ProcessRedditPost(
+ [QueueTrigger(QueueConstants.RedditPostQueueName, Connection = QueueConstants.QueueConnectionStringName)] SocialGistPostId socialGistPost,
+ TraceWriter log,
+ ExecutionContext executionContext
+ )
+ {
+ log.Info($"{FunctionName} Execution begun at {DateTime.Now}");
+
+ var config = new WebConfiguration(executionContext);
+ var logger = new FunctionLog(log, executionContext.InvocationId);
+ var objectLogger = (config.UseObjectLogger) ? new BlobObjectLogger(config, logger) : null;
+
+ using (var kernel = new KernelFactory().GetKernel(logger, config, objectLogger))
+ {
+ var processor = kernel.Get();
+ var socialGist = kernel.Get();
+ socialGist.ResultLimitPerPage = config.ResultLimitPerPage;
+ socialGist.MaximumResultsPerSearch = config.MaximumResultsPerSearch;
+
+ await processor.Process(socialGistPost);
+ }
+
+ log.Info($"{FunctionName} completed at {DateTime.Now}");
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/QueueConstants.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/QueueConstants.cs
new file mode 100644
index 000000000..e848637c0
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/QueueConstants.cs
@@ -0,0 +1,13 @@
+namespace RedditAzureFunctions
+{
+ public class QueueConstants
+ {
+ public const string RedditSearchQueueName = "%RedditSearchQueueName%";
+ public const string RedditPostQueueName = "%" + RedditPostQueueNameProperty + "%";
+ public const string RedditPostQueueNameProperty = "RedditPostQueueName";
+ public const string AzureMLJobQueueName = "%" + AzureMLJobQueueNameProperty + "%";
+ public const string AzureMLJobQueueNameProperty = "AzureMLJobQueueName";
+
+ public const string QueueConnectionStringName = "StorageQueueConnection";
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/RedditAzureFunctions.csproj b/Functions/Code/Reddit/Src/RedditAzureFunctions/RedditAzureFunctions.csproj
new file mode 100644
index 000000000..2fb1bab97
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/RedditAzureFunctions.csproj
@@ -0,0 +1,31 @@
+
+
+ net461
+
+
+ TRACE;DEBUG;NET461
+
+
+
+
+
+
+
+
+
+
+
+
+ ..\..\packages\Microsoft.ApplicationInsights.2.4.0\lib\net46\Microsoft.ApplicationInsights.dll
+
+
+
+
+
+ PreserveNewest
+
+
+ PreserveNewest
+
+
+
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/ScheduledAzureML.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/ScheduledAzureML.cs
new file mode 100644
index 000000000..6fde3cd0d
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/ScheduledAzureML.cs
@@ -0,0 +1,65 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Azure.WebJobs.Host;
+using Ninject;
+using RedditCore;
+using RedditAzureFunctions.Logging;
+using RedditCore.AzureML;
+
+namespace RedditAzureFunctions
+{
+ public static class ScheduledAzureML
+ {
+ private const string FunctionName = nameof(ScheduledAzureML);
+ private const string ScheduledAzureMLFrequencyName = "%ScheduledAzureMLFrequency%";
+
+ ///
+ /// Perform application initialization on startup
+ ///
+ static ScheduledAzureML()
+ {
+ new Bootstrap().Init();
+ }
+
+ [FunctionName(FunctionName)]
+ [Singleton(Mode = SingletonMode.Function)]
+ public static async Task Run(
+ [TimerTrigger(ScheduledAzureMLFrequencyName, RunOnStartup = WebServiceRunConstants.RunAmlOnStartup)] TimerInfo timer, // Every half hour
+ [Queue(QueueConstants.AzureMLJobQueueName, Connection = QueueConstants.QueueConnectionStringName)] ICollector queueCollector,
+ ExecutionContext executionContext,
+ TraceWriter logger
+ )
+ {
+ logger.Info($"{FunctionName} Execution begun at {DateTime.Now}");
+ IConfiguration webConfiguration = new WebConfiguration(executionContext);
+ var log = new FunctionLog(logger, executionContext.InvocationId);
+
+ var objectLogger = (webConfiguration.UseObjectLogger) ? new BlobObjectLogger(webConfiguration, log) : null;
+
+ using (var kernel = new KernelFactory().GetKernel(
+ log,
+ webConfiguration,
+ objectLogger
+ ))
+ {
+
+ var processor = kernel.Get();
+ var result = await processor.RunAzureMLProcessing();
+
+ // If result is null then there is not any data to run through AzureML and no AML job was started.
+ if (result != null)
+ {
+ queueCollector.Add(result.JobId);
+ log.Verbose($"AzureML Web Service called; JobId=[{result.JobId}]");
+ }
+ else
+ {
+ log.Verbose("No data to run through AzureML; no AML job started.");
+ }
+ }
+
+ logger.Info($"{FunctionName} completed at {DateTime.Now}");
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/ScheduledRedditSearch.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/ScheduledRedditSearch.cs
new file mode 100644
index 000000000..5cc8ec49e
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/ScheduledRedditSearch.cs
@@ -0,0 +1,33 @@
+using System;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Azure.WebJobs.Host;
+
+namespace RedditAzureFunctions
+{
+ public static class ScheduledRedditSearch
+ {
+ private const string FunctionName = nameof(ScheduledRedditSearch);
+ private const string ScheduledRedditQueryFrequencyName = "%ScheduledRedditQueryFrequency%";
+
+ ///
+ /// Perform application initialization on startup
+ ///
+ static ScheduledRedditSearch()
+ {
+ new Bootstrap().Init();
+ }
+
+ [FunctionName(FunctionName)]
+ public static void Run(
+ [TimerTrigger(ScheduledRedditQueryFrequencyName, RunOnStartup = WebServiceRunConstants.RunRedditOnStartup)] TimerInfo timer,
+ [Queue(QueueConstants.RedditSearchQueueName, Connection = QueueConstants.QueueConnectionStringName)] ICollector queueCollector,
+ ExecutionContext executionContext,
+ TraceWriter logger
+ )
+ {
+ logger.Info($"{FunctionName} initiated by timer at {DateTime.Now}");
+ queueCollector.Add("begin");
+ logger.Info($"{FunctionName} completed at {DateTime.Now}");
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/WebConfiguration.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/WebConfiguration.cs
new file mode 100644
index 000000000..4f4abae6a
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/WebConfiguration.cs
@@ -0,0 +1,105 @@
+using System.Configuration;
+using RedditCore;
+using System;
+using Microsoft.Azure.WebJobs;
+
+namespace RedditAzureFunctions
+{
+ internal class WebConfiguration : IConfiguration
+ {
+ internal WebConfiguration(ExecutionContext context)
+ {
+ this.FunctionInvocationId = context.InvocationId;
+ this.FunctionName = context.FunctionName;
+ }
+
+ public string AzureMLApiKey => ConfigurationManager.AppSettings["AzureMLApiKey"];
+
+ public string AzureMLBaseUrl => ConfigurationManager.AppSettings["AzureMLBaseUrl"];
+
+ public string DbConnectionString => ConfigurationManager.ConnectionStrings["SqlConnection"].ConnectionString;
+
+ public string SocialGistApiKey => ConfigurationManager.AppSettings["SocialGistApiKey"];
+
+ public TimeSpan SocialGistApiRequestTimeout =>
+ ConfigurationManager.AppSettings["SocialGistApiRequestTimeoutInSeconds"] != null
+ ? TimeSpan.FromSeconds(
+ int.Parse(ConfigurationManager.AppSettings["SocialGistApiRequestTimeoutInSeconds"]))
+ : TimeSpan.FromSeconds(100); // HttpClient has a default timeout of 100 seconds
+
+ private const string SearchToThreadTimeDelayProperty = "SearchToThreadTimeDelayInSeconds";
+
+ public TimeSpan SearchToThreadTimeDelay =>
+ ConfigurationManager.AppSettings[SearchToThreadTimeDelayProperty] != null
+ ? TimeSpan.FromSeconds(
+ int.Parse(ConfigurationManager.AppSettings[SearchToThreadTimeDelayProperty]))
+ : TimeSpan.FromMinutes(1);
+
+ private const string AzureMlRetryTimeDelayProperty = "AzureMLRetryTimeDelay";
+
+ public TimeSpan AzureMlRetryTimeDelay =>
+ ConfigurationManager.AppSettings[AzureMlRetryTimeDelayProperty] != null
+ ? TimeSpan.FromSeconds(
+ int.Parse(ConfigurationManager.AppSettings[AzureMlRetryTimeDelayProperty]))
+ : TimeSpan.FromSeconds(20);
+
+ public string QueryTerms => ConfigurationManager.AppSettings["QueryTerms"];
+
+ ///
+ /// Choices here, from the SocialGist BoardReader API Documentation:
+ /// Defines the result sort type. Can be as follows:
+ /// ‘relevance’;
+ /// ‘time_relevance’, sorts by time segments(last hour/day/week/month) in descending order, and then by relevance in descending order;
+ /// ‘time_desc’, most recent posts first; (default)
+ /// ‘time_asc’, oldest posts first.
+ ///
+ public string QuerySortOrder => GetOrDefault("QuerySortOrder", "time_desc");
+
+ // ReSharper disable once BuiltInTypeReferenceStyle
+ public int MaximumResultsPerSearch => int.Parse(ConfigurationManager.AppSettings["MaximumResultsPerSearch"]);
+
+ // ReSharper disable once BuiltInTypeReferenceStyle
+ public int ResultLimitPerPage => int.Parse(ConfigurationManager.AppSettings["ResultLimitPerPage"]);
+
+ public string ObjectLogSASUrl => ConfigurationManager.AppSettings["ObjectLogSASUrl"];
+
+ public string ObjectLogBlobContainer => GetOrDefault("ObjectLogBlobContainer", "object-logs");
+
+ public bool UseObjectLogger => ConfigurationManager.AppSettings["ObjectLogSASUrl"] != null;
+
+ // Used solely for logging purposes
+ public string RedditPostQueueName => ConfigurationManager.AppSettings["RedditPostQueueName"];
+
+ public string AzureMLJobQueueName => ConfigurationManager.AppSettings["AzureMLJobQueueName"];
+
+ public Guid FunctionInvocationId { get; private set; }
+
+ public string FunctionName { get; private set; }
+
+ public bool IngestOnlyDocumentsWithUserDefinedEntities => GetOrDefault("IngestOnlyDocumentsWithUserDefinedEntities", true);
+
+ private static bool GetOrDefault(string property, bool defaultValue)
+ {
+ if (ConfigurationManager.AppSettings[property] != null)
+ {
+ return Boolean.Parse(ConfigurationManager.AppSettings[property]);
+ }
+ else
+ {
+ return defaultValue;
+ }
+ }
+
+ private static string GetOrDefault(string property, string defaultValue)
+ {
+ if (ConfigurationManager.AppSettings[property] != null)
+ {
+ return ConfigurationManager.AppSettings[property];
+ }
+ else
+ {
+ return defaultValue;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/WebServiceRunConstants.cs b/Functions/Code/Reddit/Src/RedditAzureFunctions/WebServiceRunConstants.cs
new file mode 100644
index 000000000..fb7244046
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/WebServiceRunConstants.cs
@@ -0,0 +1,19 @@
+namespace RedditAzureFunctions
+{
+ ///
+ /// These constants are in a separate file to prevent us accidently checking them in. They must be changed
+ /// for local debugging but we do not want to check them in to the repository. It will really mess up
+ ///
+ public static class WebServiceRunConstants
+ {
+ ///
+ /// True to run the Reddit query on startup. Falso not to run the Reddit query on host startup or deploy.
+ ///
+ public const bool RunRedditOnStartup = false;
+
+ ///
+ /// True to run AzureML on startup. False not to run AzureML on host startup or deploy.
+ ///
+ public const bool RunAmlOnStartup = false;
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/host.json b/Functions/Code/Reddit/Src/RedditAzureFunctions/host.json
new file mode 100644
index 000000000..abe88266a
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/host.json
@@ -0,0 +1,12 @@
+{
+ "functionTimeout": "00:10:00",
+ "tracing": {
+ "consoleLevel": "verbose",
+ "fileLoggingMode": "debugOnly"
+ },
+ "queues": {
+ "batchSize": 7,
+ "newBatchThreshold": 3,
+ "visibilityTimeout": "00:15:00"
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditAzureFunctions/local.settings.json b/Functions/Code/Reddit/Src/RedditAzureFunctions/local.settings.json
new file mode 100644
index 000000000..2892e9fe4
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditAzureFunctions/local.settings.json
@@ -0,0 +1,27 @@
+{
+ "IsEncrypted": false,
+ "Values": {
+ "AzureMLApiKey": "",
+ "AzureMLBaseUrl": "https://ussouthcentral.services.azureml.net/subscriptions/SUBSCRIPTIONGUID/services/SERVICEGUID/jobs?api-version=2.0",
+ "SocialGistApiKey": "",
+ "AzureWebJobsStorage": "DefaultEndpointsProtocol=https;AccountName=ACCOUNTNAME;AccountKey=ACCOUNTKEY;EndpointSuffix=core.windows.net",
+ "QueryTerms": "superman|batman|flash|\"wonder+woman\"|\"green+lantern\"|firestorm|marvel|\"dc+comics\"|\"dc+comic\"|\"justice+league\"",
+ "SearchToThreadTimeDelayInSeconds": "5",
+ "MaximumResultsPerSearch": "100",
+ "SocialGistApiRequestTimeoutInSeconds": "300",
+ "ResultLimitPerPage": "10",
+ "RedditSearchQueueName": "reddit-search-source-dev",
+ "RedditPostQueueName": "reddit-source-dev",
+ "AzureMLJobQueueName": "reddit-azureml-job-source-dev",
+ "StorageQueueConnection": "DefaultEndpointsProtocol=https;AccountName=STORAGEQUEUEACCOUNTNAME;AccountKey=STORAGEQUEUEACCOUNTKEY;EndpointSuffix=core.windows.net",
+ "ScheduledRedditQueryFrequency": "0 0 1 * * *",
+ "ScheduledAzureMLFrequency": "0 */30 * * * *"
+ },
+ "ConnectionStrings": {
+ "SqlConnection": "Data Source=(localdb)\\MSSQLLocalDB;Initial Catalog=Reddit;Integrated Security=True;Pooling=False"
+ },
+ "Host": {
+ "LocalHttpPort": 7071,
+ "CORS": "*"
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/App.config b/Functions/Code/Reddit/Src/RedditCore/App.config
new file mode 100644
index 000000000..41f5ec4d4
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/App.config
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/AzureMLExperimentRunner.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/AzureMLExperimentRunner.cs
new file mode 100644
index 000000000..4f6803a0a
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/AzureMLExperimentRunner.cs
@@ -0,0 +1,103 @@
+using System.Collections.Generic;
+using System.Data.SqlClient;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Ninject;
+using RedditCore.Logging;
+
+namespace RedditCore.AzureML
+{
+ internal class AzureMLExperimentRunner : IAzureMLExperimentRunner
+ {
+ private readonly IExperimentCompletionWaiter completionWaiter;
+ private readonly IConfiguration configuration;
+ private readonly ILog log;
+ private readonly IObjectLogger objectLogger;
+
+ [Inject]
+ public AzureMLExperimentRunner(
+ ILog log,
+ IConfiguration configuration,
+ IExperimentCompletionWaiter completionWaiter,
+ IObjectLogger objectLogger)
+ {
+ this.log = log;
+ this.configuration = configuration;
+ this.completionWaiter = completionWaiter;
+ this.objectLogger = objectLogger;
+ }
+
+ public async Task RunExperimentWithData(string data, bool waitForCompletion)
+ {
+ var globalParameters = new Dictionary();
+
+ var builder = new SqlConnectionStringBuilder(configuration.DbConnectionString);
+
+ if (data != null)
+ {
+ globalParameters.Add("Data", data);
+ }
+ globalParameters.Add("Database server name", builder.DataSource);
+ globalParameters.Add("Database name", builder.InitialCatalog);
+ globalParameters.Add("Server user account name", builder.UserID);
+ globalParameters.Add("Server user account password", builder.Password);
+
+ using (var client = new HttpClient())
+ {
+ var baseUrl = configuration.AzureMLBaseUrl.TrimEndString("?api-version=2.0");
+
+ var request = new BatchExecutionRequest
+ {
+ GlobalParameters = globalParameters
+ };
+
+ client.DefaultRequestHeaders.Authorization =
+ new AuthenticationHeaderValue("Bearer", configuration.AzureMLApiKey);
+
+ log.Info("Submitting the job...", "AzureMLExperimentRunner");
+
+ // submit the job
+ var serialized = JsonConvert.SerializeObject(request);
+
+ HttpContent httpContent = new ByteArrayContent(Encoding.ASCII.GetBytes(serialized));
+ httpContent.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+ var response = await client.PostAsync(
+ $"{baseUrl}?api-version=2.0",
+ httpContent
+ );
+
+ if (!response.IsSuccessStatusCode)
+ return new AzureMLResult {Success = false, Error = "Unable to submit job", Details = response.ToString()};
+
+ var jobIdBytes = await response.Content.ReadAsByteArrayAsync();
+
+ // Job ID is returned with quotes. Remove the quotes.
+ var jobId = Encoding.ASCII.GetString(jobIdBytes).Trim('"');
+ this.objectLogger.Log(data, "AzureMLExperimentRunner", $"Submitted jobId: {jobId}");
+ log.Info($"Submitted job ID: {jobId}", "AzureMLExperimentRunner");
+
+ // start the job
+ log.Info("Starting the job...");
+ response = await client.PostAsync(
+ $"{baseUrl}/{jobId}/start?api-version=2.0",
+ null
+ );
+ if (!response.IsSuccessStatusCode)
+ return new AzureMLResult {Success = false, Error = $"Unable to start job {jobId}"};
+
+ if (waitForCompletion)
+ return await completionWaiter.WaitForJobCompletion(jobId);
+ return new AzureMLResult
+ {
+ // Set success to null because we did not wait for completion. We do not know if the job was successful or not.
+ Success = null,
+ LastJobStatus = BatchScoreStatusCode.Unknown,
+ JobId = jobId
+ };
+ }
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/AzureMLResult.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/AzureMLResult.cs
new file mode 100644
index 000000000..47f8f8b1c
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/AzureMLResult.cs
@@ -0,0 +1,27 @@
+namespace RedditCore.AzureML
+{
+ public class AzureMLResult
+ {
+ ///
+ /// True if the job ran successfully, false if there was a failure, or null
+ /// if the job was sucessfully submitted but we were asked not to wait for completion.
+ ///
+ public bool? Success { get; internal set; }
+
+ public string Message { get; internal set; }
+
+ public string Error { get; internal set; }
+
+ public string Details { get; internal set; }
+
+ ///
+ /// Gets the ID of the AzureML job
+ ///
+ public string JobId { get; internal set; }
+
+ ///
+ /// Gets the last job status of the AzureML job. This may not be the final status of the job. This is the last status that the experiment waiter code saw.
+ ///
+ public BatchScoreStatusCode LastJobStatus { get; internal set; }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchExecutionRequest.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchExecutionRequest.cs
new file mode 100644
index 000000000..6976148b0
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchExecutionRequest.cs
@@ -0,0 +1,14 @@
+using System.Collections.Generic;
+
+namespace RedditCore.AzureML
+{
+ internal class BatchExecutionRequest
+ {
+ public IDictionary Inputs { get; set; }
+
+ public IDictionary GlobalParameters { get; set; }
+
+ // Locations for the potential multiple batch scoring outputs
+ public IDictionary Outputs { get; set; }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchScoreStatus.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchScoreStatus.cs
new file mode 100644
index 000000000..2f75ba3fb
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchScoreStatus.cs
@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+
+namespace RedditCore.AzureML
+{
+ internal class BatchScoreStatus
+ {
+ // Status code for the batch scoring job
+ public BatchScoreStatusCode StatusCode { get; set; }
+
+ // Locations for the potential multiple batch scoring outputs
+ public IDictionary Results { get; set; }
+
+ // Error details, if any
+ public string Details { get; set; }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchScoreStatusCode.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchScoreStatusCode.cs
new file mode 100644
index 000000000..59cf969a2
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/BatchScoreStatusCode.cs
@@ -0,0 +1,21 @@
+namespace RedditCore.AzureML
+{
+ public enum BatchScoreStatusCode
+ {
+ Unknown,
+ NotStarted,
+ Running,
+ Failed,
+ Cancelled,
+ Finished
+ }
+ public static class BatchScoreStatusCodeExtensions
+ {
+ public static bool IsTerminalState(this BatchScoreStatusCode code)
+ {
+ return code == BatchScoreStatusCode.Finished
+ || code == BatchScoreStatusCode.Failed
+ || code == BatchScoreStatusCode.Cancelled;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/ExperimentCompletionWaiter.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/ExperimentCompletionWaiter.cs
new file mode 100644
index 000000000..0dbc3483d
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/ExperimentCompletionWaiter.cs
@@ -0,0 +1,124 @@
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Ninject;
+using RedditCore.Logging;
+using RedditCore.Http;
+using System.Net.Http.Headers;
+using System;
+
+namespace RedditCore.AzureML
+{
+ internal class ExperimentCompletionWaiter : IExperimentCompletionWaiter
+ {
+ private readonly IConfiguration configuration;
+ private readonly ILog log;
+ private readonly IHttpClient httpClient;
+
+ [Inject]
+ public ExperimentCompletionWaiter(
+ ILog log,
+ IConfiguration configuration,
+ IHttpClient client)
+ {
+ this.configuration = configuration;
+ this.log = log;
+ this.httpClient = client;
+ }
+
+ public async Task WaitForJobCompletion(string jobId)
+ {
+ return await WaitForJobCompletion(jobId, TimeSpan.FromMinutes(10), true);
+ }
+
+ public async Task WaitForJobCompletion(string jobId, TimeSpan timeout, bool killJobAfterTimeout)
+ {
+ var jobLocation = new Uri(configuration.AzureMLBaseUrl.TrimEndString("?api-version=2.0") + "/" + jobId + "?api-version=2.0");
+ var done = false;
+
+ bool? success = false;
+ var message = "";
+ var details = "";
+
+ // Set a default. "Unknown" is not a valid response and AzureML will never return it.
+ BatchScoreStatusCode lastStatus = BatchScoreStatusCode.Unknown;
+
+ var watch = Stopwatch.StartNew();
+
+ var authenticationHeaderValue = new AuthenticationHeaderValue("Bearer", configuration.AzureMLApiKey);
+
+ while (!done)
+ {
+ log.Info("Checking the job status...");
+ var response = await httpClient.GetJsonAsync(
+ jobLocation,
+ authenticationHeaderValue: authenticationHeaderValue);
+
+ if (!response.ResponseMessage.IsSuccessStatusCode)
+ return new AzureMLResult { Success = false, Error = $"Error in Job ID {jobId}" };
+
+ if (watch.ElapsedMilliseconds > timeout.TotalMilliseconds)
+ {
+ if (killJobAfterTimeout)
+ {
+ done = true;
+ log.Info(string.Format("Timed out. Deleting job {0} ...", jobId));
+ message = "ExperimentCompletionWaiter killed the job due to timeout";
+ await httpClient.DeleteAsync(jobLocation, authenticationHeaderValue: authenticationHeaderValue);
+ }
+ else
+ {
+ done = true;
+ message = "ExperimentCompletionWaiter timed out without canceling the job";
+ log.Info($"Timed out. Leaving job {jobId} running ...");
+ }
+ }
+
+ var status = response.Object;
+ lastStatus = status.StatusCode;
+ log.Verbose($"Job {jobId} status {status.StatusCode}");
+
+ switch (status.StatusCode)
+ {
+ case BatchScoreStatusCode.NotStarted:
+ log.Info(string.Format("Job {0} not yet started...", jobId));
+ break;
+ case BatchScoreStatusCode.Running:
+ log.Info(string.Format("Job {0} running...", jobId));
+ break;
+ case BatchScoreStatusCode.Failed:
+ log.Info(string.Format("Job {0} failed!", jobId));
+ log.Info(string.Format("Error details: {0}", status.Details));
+ done = true;
+ message = $"Job {jobId} failed";
+ details = status.Details;
+ break;
+ case BatchScoreStatusCode.Cancelled:
+ log.Info(string.Format("Job {0} cancelled!", jobId));
+ done = true;
+ message = $"Cancelled job {jobId}";
+ break;
+ case BatchScoreStatusCode.Finished:
+ done = true;
+ log.Info(string.Format("Job {0} finished!", jobId));
+ log.Info("Response: ");
+ log.Info(await response.ResponseMessage.Content.ReadAsStringAsync());
+ success = true;
+ break;
+ }
+
+ if (!done)
+ Thread.Sleep(1000); // Wait one second
+ }
+
+ return new AzureMLResult
+ {
+ Success = success,
+ Message = message,
+ Details = details,
+ LastJobStatus = lastStatus,
+ JobId = jobId
+ };
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/IAzureMLExperimentRunner.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/IAzureMLExperimentRunner.cs
new file mode 100644
index 000000000..83895596a
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/IAzureMLExperimentRunner.cs
@@ -0,0 +1,9 @@
+using System.Threading.Tasks;
+
+namespace RedditCore.AzureML
+{
+ public interface IAzureMLExperimentRunner
+ {
+ Task RunExperimentWithData(string data, bool waitForCompletion);
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/IExperimentCompletionWaiter.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/IExperimentCompletionWaiter.cs
new file mode 100644
index 000000000..c0b45673f
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/IExperimentCompletionWaiter.cs
@@ -0,0 +1,24 @@
+using System;
+using System.Threading.Tasks;
+
+namespace RedditCore.AzureML
+{
+ public interface IExperimentCompletionWaiter
+ {
+ ///
+ /// Waits for a job to complete. If the job does not complete after 10 minutes then the job will be killed.
+ ///
+ /// ID of the job to wait for
+ /// Result of the job run
+ Task WaitForJobCompletion(string jobId);
+
+ ///
+ /// Waits for a job to complete.
+ ///
+ /// ID of the job to wait for
+ /// Time to wait for the job to complete
+ /// If true after the timeout has elapsed then the job will be killed. If false after the timeout has elapsed then the function will exit and the job will be left running.
+ /// Result of the job run
+ Task WaitForJobCompletion(string jobId, TimeSpan timeout, bool killJobAfterTimeout);
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/IScheduledAzureMLProcessor.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/IScheduledAzureMLProcessor.cs
new file mode 100644
index 000000000..c6e8d76ab
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/IScheduledAzureMLProcessor.cs
@@ -0,0 +1,20 @@
+using System.Threading.Tasks;
+
+namespace RedditCore.AzureML
+{
+ public interface IScheduledAzureMLProcessor
+ {
+ ///
+ /// Start the AzureML job
+ ///
+ /// Result of the job start
+ Task RunAzureMLProcessing();
+
+ ///
+ /// Checks an AzureML job. If it finishes or is finished then run the post-process cleanup.
+ ///
+ /// ID of the job to run
+ /// Result of the job run
+ Task CheckAzureMLAndPostProcess(string jobId);
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/ScheduledAzureMLProcessor.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/ScheduledAzureMLProcessor.cs
new file mode 100644
index 000000000..6b65e77c7
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/ScheduledAzureMLProcessor.cs
@@ -0,0 +1,112 @@
+using RedditCore.Logging;
+using System;
+using System.Data;
+using System.Threading.Tasks;
+using RedditCore.DataModel.Repositories;
+using RedditCore.Properties;
+using RedditCore.Telemetry;
+
+namespace RedditCore.AzureML
+{
+ internal class ScheduledAzureMLProcessor : IScheduledAzureMLProcessor
+ {
+ private readonly IAzureMLExperimentRunner experimentRunner;
+ private readonly IExperimentCompletionWaiter completionWaiter;
+ private readonly IDbConnectionFactory connectionFactory;
+ private readonly ITelemetryClient telemetryClient;
+ private readonly ILog log;
+
+ public ScheduledAzureMLProcessor(
+ IAzureMLExperimentRunner experimentRunner,
+ IExperimentCompletionWaiter waiter,
+ IDbConnectionFactory connectionFactory,
+ ITelemetryClient telemetryClient,
+ ILog log)
+ {
+ this.experimentRunner = experimentRunner;
+ this.connectionFactory = connectionFactory;
+ this.log = log;
+ this.completionWaiter = waiter;
+ this.telemetryClient = telemetryClient;
+ }
+
+ public async Task RunAzureMLProcessing()
+ {
+ if (DataReadyForAzureMachineLearning())
+ {
+ this.telemetryClient.TrackEvent(TelemetryNames.ScheduledAzureMLProcessor_DataFound);
+
+ var azureMLResult = await experimentRunner.RunExperimentWithData(null, false);
+
+ if (!azureMLResult.Success.GetValueOrDefault(true))
+ throw new ScheduledAzureMLProcessorException(
+ $"Error in AzureML pipeline: {azureMLResult.Error} Details: {azureMLResult.Details}");
+
+ return azureMLResult;
+ }
+ else
+ {
+ this.telemetryClient.TrackEvent(TelemetryNames.ScheduledAzureMLProcessor_NoDataFound);
+ return null;
+ }
+ }
+
+ private bool DataReadyForAzureMachineLearning()
+ {
+ using (var connection = connectionFactory.CreateDbConnection())
+ {
+ connection.Open();
+
+ // SELECT a null row for each row in AzureMachineLearningInputView.
+ // If any rows are found then return 1. If no rows are found then return 0.
+ // This was done to try and avoid SELECT COUNT(*) FROM AzureMachineLearningInputView and then checking the count.
+ const string query = @"
+ SELECT (
+ CASE WHEN EXISTS(SELECT TOP 5 NULL FROM [reddit].[AzureMachineLearningInputView])
+ THEN 1
+ ELSE 0
+ END
+ ) AS isNotEmpty";
+
+ using (var command = connection.CreateCommand())
+ {
+ command.CommandText = query;
+ command.CommandType = CommandType.Text;
+
+ var result = command.ExecuteScalar();
+
+ // If the command returns 1 then we have data to process!
+ return result?.ToString() == "1";
+ }
+ }
+ }
+
+ public async Task CheckAzureMLAndPostProcess(string jobId)
+ {
+ var azureMlResult = await completionWaiter.WaitForJobCompletion(jobId, TimeSpan.Zero, false);
+
+ if (!azureMlResult.LastJobStatus.IsTerminalState()) return azureMlResult;
+ log.Info("AzureML completed. Ready for staging migration");
+
+ using (var connection = connectionFactory.CreateDbConnection())
+ {
+ connection.Open();
+
+ using (var command = connection.CreateCommand())
+ {
+ // Set the timeout to five minutes. This may be a long task
+ command.CommandTimeout = 300;
+
+ command.CommandText = "reddit.PostAzureML";
+ command.CommandType = CommandType.StoredProcedure;
+
+ log.Info("Pushing to live tables from staging");
+ int result = command.ExecuteNonQuery();
+ log.Info($"Pushed to live tables from staging. Stored Procedure result: {result}");
+ }
+ }
+
+ return azureMlResult;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/AzureML/ScheduledAzureMLProcessorException.cs b/Functions/Code/Reddit/Src/RedditCore/AzureML/ScheduledAzureMLProcessorException.cs
new file mode 100644
index 000000000..3f6a74fcd
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/AzureML/ScheduledAzureMLProcessorException.cs
@@ -0,0 +1,19 @@
+using System;
+
+namespace RedditCore.AzureML
+{
+ public class ScheduledAzureMLProcessorException : Exception
+ {
+ public ScheduledAzureMLProcessorException()
+ : base()
+ {
+
+ }
+
+ public ScheduledAzureMLProcessorException(string message)
+ : base(message)
+ {
+
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Comment.Partial.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Comment.Partial.cs
new file mode 100644
index 000000000..95b50f564
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Comment.Partial.cs
@@ -0,0 +1,30 @@
+using System.Reflection;
+using System.Text;
+
+namespace RedditCore.DataModel
+{
+ public partial class Comment : IDocument
+ {
+ public override string ToString()
+ {
+ var flags = BindingFlags.Instance | BindingFlags.Public |
+ BindingFlags.FlattenHierarchy;
+ var infos = GetType().GetProperties(flags);
+
+ var sb = new StringBuilder();
+
+ var typeName = GetType().Name;
+ sb.Append(typeName);
+
+ sb.Append("[");
+ foreach (var info in infos)
+ {
+ var value = info.GetValue(this, null);
+ sb.AppendFormat("{0}: {1},", info.Name, value != null ? value : "null");
+ }
+ sb.Append("]");
+
+ return sb.ToString();
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Comment.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Comment.cs
new file mode 100644
index 000000000..97fccd675
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Comment.cs
@@ -0,0 +1,67 @@
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
+namespace RedditCore.DataModel
+{
+
+using System;
+ using System.Collections.Generic;
+
+public partial class Comment
+{
+
+ public string Id { get; set; }
+
+ public string Content { get; set; }
+
+ public Nullable Gilded { get; set; }
+
+ public string Author { get; set; }
+
+ public string Subreddit { get; set; }
+
+ public Nullable IsComment { get; private set; }
+
+ public Nullable Sentiment { get; private set; }
+
+ public string ParentId { get; set; }
+
+ public System.DateTime PublishedTimestamp { get; set; }
+
+ public System.DateTime PublishedMonthPrecision { get; set; }
+
+ public System.DateTime PublishedWeekPrecision { get; set; }
+
+ public System.DateTime PublishedDayPrecision { get; set; }
+
+ public System.DateTime PublishedHourPrecision { get; set; }
+
+ public System.DateTime PublishedMinutePrecision { get; set; }
+
+ public System.DateTime IngestedTimestamp { get; set; }
+
+ public string Url { get; set; }
+
+ public string PostId { get; set; }
+
+ public Nullable Score { get; set; }
+
+ public Nullable Controversiality { get; set; }
+
+ public string ParentUrl { get; set; }
+
+ public string PostUrl { get; set; }
+
+ public Nullable DocumentContainsUserDefinedEntities { get; private set; }
+
+}
+
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/DataModelModule.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/DataModelModule.cs
new file mode 100644
index 000000000..1c9cea0b2
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/DataModelModule.cs
@@ -0,0 +1,25 @@
+using System.Data.Entity.Core.EntityClient;
+using Ninject.Modules;
+using RedditCore.DataModel.Repositories;
+
+namespace RedditCore.DataModel
+{
+ public class DataModelModule : NinjectModule
+ {
+ public override void Load()
+ {
+ Bind>().To();
+ Bind>().To();
+ Bind>().To();
+ Bind>().To();
+ Bind>().To();
+ Bind>().To();
+ Bind().To();
+
+ // Singleton scope is needed to make sure this is disposed when the kernel is disposed
+ Bind().ToProvider().InSingletonScope();
+
+ Bind().To().InSingletonScope();
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/EmbeddedUrl.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/EmbeddedUrl.cs
new file mode 100644
index 000000000..4116a04d2
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/EmbeddedUrl.cs
@@ -0,0 +1,31 @@
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
+namespace RedditCore.DataModel
+{
+
+using System;
+ using System.Collections.Generic;
+
+public partial class EmbeddedUrl
+{
+
+ public long Id { get; set; }
+
+ public string DocumentId { get; set; }
+
+ public string ContentEmbeddedUrl { get; set; }
+
+ public string ContentEmbeddedUrlDomain { get; set; }
+
+}
+
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/EntityConnectionProvider.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/EntityConnectionProvider.cs
new file mode 100644
index 000000000..af07e5cbf
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/EntityConnectionProvider.cs
@@ -0,0 +1,27 @@
+using System.Data.Entity.Core.EntityClient;
+using Ninject;
+using Ninject.Activation;
+
+namespace RedditCore.DataModel
+{
+ internal class EntityConnectionProvider : Provider
+ {
+ private readonly IConfiguration configuration;
+
+ [Inject]
+ public EntityConnectionProvider(IConfiguration configuration)
+ {
+ this.configuration = configuration;
+ }
+
+ protected override EntityConnection CreateInstance(IContext context)
+ {
+ var connectionBuilder = new EntityConnectionStringBuilder();
+ connectionBuilder.Provider = "System.Data.SqlClient";
+ connectionBuilder.ProviderConnectionString = configuration.DbConnectionString;
+ connectionBuilder.Metadata = "res://*/DataModel.Reddit.csdl|res://*/DataModel.Reddit.ssdl|res://*/DataModel.Reddit.msl";
+
+ return new EntityConnection(connectionBuilder.ToString());
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/IDocument.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/IDocument.cs
new file mode 100644
index 000000000..84beed43c
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/IDocument.cs
@@ -0,0 +1,9 @@
+namespace RedditCore.DataModel
+{
+ public interface IDocument
+ {
+ string Id { get; }
+
+ string Content { get; }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/InvalidComment.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/InvalidComment.cs
new file mode 100644
index 000000000..7fe05458d
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/InvalidComment.cs
@@ -0,0 +1,9 @@
+namespace RedditCore.DataModel
+{
+ ///
+ /// Marker class for invalid comments that we should remove from the database.
+ ///
+ internal class InvalidComment : Comment
+ {
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Post.Partial.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Post.Partial.cs
new file mode 100644
index 000000000..b7daa3011
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Post.Partial.cs
@@ -0,0 +1,30 @@
+using System.Reflection;
+using System.Text;
+
+namespace RedditCore.DataModel
+{
+ public partial class Post : IDocument
+ {
+ public override string ToString()
+ {
+ var flags = BindingFlags.Instance | BindingFlags.Public |
+ BindingFlags.FlattenHierarchy;
+ var infos = GetType().GetProperties(flags);
+
+ var sb = new StringBuilder();
+
+ var typeName = GetType().Name;
+ sb.Append(typeName);
+
+ sb.Append("[");
+ foreach (var info in infos)
+ {
+ var value = info.GetValue(this, null);
+ sb.AppendFormat("{0}: {1},", info.Name, value != null ? value : "null");
+ }
+ sb.Append("]");
+
+ return sb.ToString();
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Post.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Post.cs
new file mode 100644
index 000000000..c3d0c0134
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Post.cs
@@ -0,0 +1,63 @@
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
+namespace RedditCore.DataModel
+{
+
+using System;
+ using System.Collections.Generic;
+
+public partial class Post
+{
+
+ public string Id { get; set; }
+
+ public string Content { get; set; }
+
+ public Nullable Gilded { get; set; }
+
+ public string Author { get; set; }
+
+ public string Subreddit { get; set; }
+
+ public Nullable IsComment { get; set; }
+
+ public Nullable Sentiment { get; set; }
+
+ public string Title { get; set; }
+
+ public System.DateTime PublishedTimestamp { get; set; }
+
+ public System.DateTime PublishedMonthPrecision { get; set; }
+
+ public System.DateTime PublishedWeekPrecision { get; set; }
+
+ public System.DateTime PublishedDayPrecision { get; set; }
+
+ public System.DateTime PublishedHourPrecision { get; set; }
+
+ public System.DateTime PublishedMinutePrecision { get; set; }
+
+ public System.DateTime IngestedTimestamp { get; set; }
+
+ public string Url { get; set; }
+
+ public string MediaPreviewUrl { get; set; }
+
+ public Nullable Score { get; set; }
+
+ public Nullable Controversiality { get; set; }
+
+ public Nullable DocumentContainsUserDefinedEntities { get; private set; }
+
+}
+
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/PostCommentCount.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/PostCommentCount.cs
new file mode 100644
index 000000000..bf9d600f2
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/PostCommentCount.cs
@@ -0,0 +1,27 @@
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
+namespace RedditCore.DataModel
+{
+
+using System;
+ using System.Collections.Generic;
+
+public partial class PostCommentCount
+{
+
+ public string PostId { get; set; }
+
+ public Nullable CommentCount { get; set; }
+
+}
+
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.Partial.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.Partial.cs
new file mode 100644
index 000000000..e592fc314
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.Partial.cs
@@ -0,0 +1,15 @@
+using System.Data.Entity.Core.EntityClient;
+using Ninject;
+
+namespace RedditCore.DataModel
+{
+ public partial class RedditEntities
+ {
+ [Inject]
+ public RedditEntities(EntityConnection conn)
+ : base(conn, false)
+ {
+
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.cs
new file mode 100644
index 000000000..5d89127f1
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.cs
@@ -0,0 +1,388 @@
+
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
+namespace RedditCore.DataModel
+{
+
+using System;
+using System.Data.Entity;
+using System.Data.Entity.Infrastructure;
+
+using System.Data.Entity.Core.Objects;
+using System.Linq;
+
+
+public partial class RedditEntities : DbContext
+{
+ public RedditEntities()
+ : base("name=RedditEntities")
+ {
+
+ }
+
+ protected override void OnModelCreating(DbModelBuilder modelBuilder)
+ {
+ throw new UnintentionalCodeFirstException();
+ }
+
+
+ public virtual DbSet Comments { get; set; }
+
+ public virtual DbSet Posts { get; set; }
+
+ public virtual DbSet UserDefinedEntities { get; set; }
+
+ public virtual DbSet UserDefinedEntityDefinitions { get; set; }
+
+ public virtual DbSet EmbeddedUrls { get; set; }
+
+ public virtual DbSet PostCommentCounts { get; set; }
+
+
+ public virtual int InsertSubreddit(Nullable newDocId, string content, Nullable ups, Nullable downs, Nullable gilded, string author, string subreddit, string title)
+ {
+
+ var newDocIdParameter = newDocId.HasValue ?
+ new ObjectParameter("NewDocId", newDocId) :
+ new ObjectParameter("NewDocId", typeof(long));
+
+
+ var contentParameter = content != null ?
+ new ObjectParameter("content", content) :
+ new ObjectParameter("content", typeof(string));
+
+
+ var upsParameter = ups.HasValue ?
+ new ObjectParameter("ups", ups) :
+ new ObjectParameter("ups", typeof(int));
+
+
+ var downsParameter = downs.HasValue ?
+ new ObjectParameter("downs", downs) :
+ new ObjectParameter("downs", typeof(int));
+
+
+ var gildedParameter = gilded.HasValue ?
+ new ObjectParameter("gilded", gilded) :
+ new ObjectParameter("gilded", typeof(int));
+
+
+ var authorParameter = author != null ?
+ new ObjectParameter("author", author) :
+ new ObjectParameter("author", typeof(string));
+
+
+ var subredditParameter = subreddit != null ?
+ new ObjectParameter("subreddit", subreddit) :
+ new ObjectParameter("subreddit", typeof(string));
+
+
+ var titleParameter = title != null ?
+ new ObjectParameter("title", title) :
+ new ObjectParameter("title", typeof(string));
+
+
+ return ((IObjectContextAdapter)this).ObjectContext.ExecuteFunction("InsertSubreddit", newDocIdParameter, contentParameter, upsParameter, downsParameter, gildedParameter, authorParameter, subredditParameter, titleParameter);
+ }
+
+
+ public virtual int InsertComment(Nullable newDocId, string content, Nullable ups, Nullable downs, Nullable gilded, string author, string subreddit, string parentId)
+ {
+
+ var newDocIdParameter = newDocId.HasValue ?
+ new ObjectParameter("NewDocId", newDocId) :
+ new ObjectParameter("NewDocId", typeof(long));
+
+
+ var contentParameter = content != null ?
+ new ObjectParameter("content", content) :
+ new ObjectParameter("content", typeof(string));
+
+
+ var upsParameter = ups.HasValue ?
+ new ObjectParameter("ups", ups) :
+ new ObjectParameter("ups", typeof(int));
+
+
+ var downsParameter = downs.HasValue ?
+ new ObjectParameter("downs", downs) :
+ new ObjectParameter("downs", typeof(int));
+
+
+ var gildedParameter = gilded.HasValue ?
+ new ObjectParameter("gilded", gilded) :
+ new ObjectParameter("gilded", typeof(int));
+
+
+ var authorParameter = author != null ?
+ new ObjectParameter("author", author) :
+ new ObjectParameter("author", typeof(string));
+
+
+ var subredditParameter = subreddit != null ?
+ new ObjectParameter("subreddit", subreddit) :
+ new ObjectParameter("subreddit", typeof(string));
+
+
+ var parentIdParameter = parentId != null ?
+ new ObjectParameter("parentId", parentId) :
+ new ObjectParameter("parentId", typeof(string));
+
+
+ return ((IObjectContextAdapter)this).ObjectContext.ExecuteFunction("InsertComment", newDocIdParameter, contentParameter, upsParameter, downsParameter, gildedParameter, authorParameter, subredditParameter, parentIdParameter);
+ }
+
+
+ public virtual int InsertSubreddit1(Nullable newDocId, string content, Nullable ups, Nullable downs, Nullable gilded, string author, string subreddit, string title)
+ {
+
+ var newDocIdParameter = newDocId.HasValue ?
+ new ObjectParameter("NewDocId", newDocId) :
+ new ObjectParameter("NewDocId", typeof(long));
+
+
+ var contentParameter = content != null ?
+ new ObjectParameter("content", content) :
+ new ObjectParameter("content", typeof(string));
+
+
+ var upsParameter = ups.HasValue ?
+ new ObjectParameter("ups", ups) :
+ new ObjectParameter("ups", typeof(int));
+
+
+ var downsParameter = downs.HasValue ?
+ new ObjectParameter("downs", downs) :
+ new ObjectParameter("downs", typeof(int));
+
+
+ var gildedParameter = gilded.HasValue ?
+ new ObjectParameter("gilded", gilded) :
+ new ObjectParameter("gilded", typeof(int));
+
+
+ var authorParameter = author != null ?
+ new ObjectParameter("author", author) :
+ new ObjectParameter("author", typeof(string));
+
+
+ var subredditParameter = subreddit != null ?
+ new ObjectParameter("subreddit", subreddit) :
+ new ObjectParameter("subreddit", typeof(string));
+
+
+ var titleParameter = title != null ?
+ new ObjectParameter("title", title) :
+ new ObjectParameter("title", typeof(string));
+
+
+ return ((IObjectContextAdapter)this).ObjectContext.ExecuteFunction("InsertSubreddit1", newDocIdParameter, contentParameter, upsParameter, downsParameter, gildedParameter, authorParameter, subredditParameter, titleParameter);
+ }
+
+
+ public virtual int InsertComment1(string newDocId, string content, Nullable score, Nullable controversiality, Nullable gilded, string author, string subreddit, Nullable publishedTimestamp, Nullable publishedMonthPrecision, Nullable publishedWeekPrecision, Nullable publishedDayPrecision, Nullable publishedHourPrecision, Nullable publishedMinutePrecision, Nullable ingestedTimestamp, string url, string parentId, string postId, string parentUrl, string postUrl)
+ {
+
+ var newDocIdParameter = newDocId != null ?
+ new ObjectParameter("NewDocId", newDocId) :
+ new ObjectParameter("NewDocId", typeof(string));
+
+
+ var contentParameter = content != null ?
+ new ObjectParameter("content", content) :
+ new ObjectParameter("content", typeof(string));
+
+
+ var scoreParameter = score.HasValue ?
+ new ObjectParameter("score", score) :
+ new ObjectParameter("score", typeof(int));
+
+
+ var controversialityParameter = controversiality.HasValue ?
+ new ObjectParameter("controversiality", controversiality) :
+ new ObjectParameter("controversiality", typeof(double));
+
+
+ var gildedParameter = gilded.HasValue ?
+ new ObjectParameter("gilded", gilded) :
+ new ObjectParameter("gilded", typeof(int));
+
+
+ var authorParameter = author != null ?
+ new ObjectParameter("author", author) :
+ new ObjectParameter("author", typeof(string));
+
+
+ var subredditParameter = subreddit != null ?
+ new ObjectParameter("subreddit", subreddit) :
+ new ObjectParameter("subreddit", typeof(string));
+
+
+ var publishedTimestampParameter = publishedTimestamp.HasValue ?
+ new ObjectParameter("publishedTimestamp", publishedTimestamp) :
+ new ObjectParameter("publishedTimestamp", typeof(System.DateTime));
+
+
+ var publishedMonthPrecisionParameter = publishedMonthPrecision.HasValue ?
+ new ObjectParameter("publishedMonthPrecision", publishedMonthPrecision) :
+ new ObjectParameter("publishedMonthPrecision", typeof(System.DateTime));
+
+
+ var publishedWeekPrecisionParameter = publishedWeekPrecision.HasValue ?
+ new ObjectParameter("publishedWeekPrecision", publishedWeekPrecision) :
+ new ObjectParameter("publishedWeekPrecision", typeof(System.DateTime));
+
+
+ var publishedDayPrecisionParameter = publishedDayPrecision.HasValue ?
+ new ObjectParameter("publishedDayPrecision", publishedDayPrecision) :
+ new ObjectParameter("publishedDayPrecision", typeof(System.DateTime));
+
+
+ var publishedHourPrecisionParameter = publishedHourPrecision.HasValue ?
+ new ObjectParameter("publishedHourPrecision", publishedHourPrecision) :
+ new ObjectParameter("publishedHourPrecision", typeof(System.DateTime));
+
+
+ var publishedMinutePrecisionParameter = publishedMinutePrecision.HasValue ?
+ new ObjectParameter("publishedMinutePrecision", publishedMinutePrecision) :
+ new ObjectParameter("publishedMinutePrecision", typeof(System.DateTime));
+
+
+ var ingestedTimestampParameter = ingestedTimestamp.HasValue ?
+ new ObjectParameter("ingestedTimestamp", ingestedTimestamp) :
+ new ObjectParameter("ingestedTimestamp", typeof(System.DateTime));
+
+
+ var urlParameter = url != null ?
+ new ObjectParameter("url", url) :
+ new ObjectParameter("url", typeof(string));
+
+
+ var parentIdParameter = parentId != null ?
+ new ObjectParameter("parentId", parentId) :
+ new ObjectParameter("parentId", typeof(string));
+
+
+ var postIdParameter = postId != null ?
+ new ObjectParameter("postId", postId) :
+ new ObjectParameter("postId", typeof(string));
+
+
+ var parentUrlParameter = parentUrl != null ?
+ new ObjectParameter("parentUrl", parentUrl) :
+ new ObjectParameter("parentUrl", typeof(string));
+
+
+ var postUrlParameter = postUrl != null ?
+ new ObjectParameter("postUrl", postUrl) :
+ new ObjectParameter("postUrl", typeof(string));
+
+
+ return ((IObjectContextAdapter)this).ObjectContext.ExecuteFunction("InsertComment1", newDocIdParameter, contentParameter, scoreParameter, controversialityParameter, gildedParameter, authorParameter, subredditParameter, publishedTimestampParameter, publishedMonthPrecisionParameter, publishedWeekPrecisionParameter, publishedDayPrecisionParameter, publishedHourPrecisionParameter, publishedMinutePrecisionParameter, ingestedTimestampParameter, urlParameter, parentIdParameter, postIdParameter, parentUrlParameter, postUrlParameter);
+ }
+
+
+ public virtual int InsertPost(string newDocId, string content, Nullable score, Nullable controversiality, Nullable gilded, string author, string subreddit, Nullable publishedTimestamp, Nullable publishedMonthPrecision, Nullable publishedWeekPrecision, Nullable publishedDayPrecision, Nullable publishedHourPrecision, Nullable publishedMinutePrecision, Nullable ingestedTimestamp, string url, string title, string mediaPreviewUrl)
+ {
+
+ var newDocIdParameter = newDocId != null ?
+ new ObjectParameter("NewDocId", newDocId) :
+ new ObjectParameter("NewDocId", typeof(string));
+
+
+ var contentParameter = content != null ?
+ new ObjectParameter("content", content) :
+ new ObjectParameter("content", typeof(string));
+
+
+ var scoreParameter = score.HasValue ?
+ new ObjectParameter("score", score) :
+ new ObjectParameter("score", typeof(int));
+
+
+ var controversialityParameter = controversiality.HasValue ?
+ new ObjectParameter("controversiality", controversiality) :
+ new ObjectParameter("controversiality", typeof(double));
+
+
+ var gildedParameter = gilded.HasValue ?
+ new ObjectParameter("gilded", gilded) :
+ new ObjectParameter("gilded", typeof(int));
+
+
+ var authorParameter = author != null ?
+ new ObjectParameter("author", author) :
+ new ObjectParameter("author", typeof(string));
+
+
+ var subredditParameter = subreddit != null ?
+ new ObjectParameter("subreddit", subreddit) :
+ new ObjectParameter("subreddit", typeof(string));
+
+
+ var publishedTimestampParameter = publishedTimestamp.HasValue ?
+ new ObjectParameter("publishedTimestamp", publishedTimestamp) :
+ new ObjectParameter("publishedTimestamp", typeof(System.DateTime));
+
+
+ var publishedMonthPrecisionParameter = publishedMonthPrecision.HasValue ?
+ new ObjectParameter("publishedMonthPrecision", publishedMonthPrecision) :
+ new ObjectParameter("publishedMonthPrecision", typeof(System.DateTime));
+
+
+ var publishedWeekPrecisionParameter = publishedWeekPrecision.HasValue ?
+ new ObjectParameter("publishedWeekPrecision", publishedWeekPrecision) :
+ new ObjectParameter("publishedWeekPrecision", typeof(System.DateTime));
+
+
+ var publishedDayPrecisionParameter = publishedDayPrecision.HasValue ?
+ new ObjectParameter("publishedDayPrecision", publishedDayPrecision) :
+ new ObjectParameter("publishedDayPrecision", typeof(System.DateTime));
+
+
+ var publishedHourPrecisionParameter = publishedHourPrecision.HasValue ?
+ new ObjectParameter("publishedHourPrecision", publishedHourPrecision) :
+ new ObjectParameter("publishedHourPrecision", typeof(System.DateTime));
+
+
+ var publishedMinutePrecisionParameter = publishedMinutePrecision.HasValue ?
+ new ObjectParameter("publishedMinutePrecision", publishedMinutePrecision) :
+ new ObjectParameter("publishedMinutePrecision", typeof(System.DateTime));
+
+
+ var ingestedTimestampParameter = ingestedTimestamp.HasValue ?
+ new ObjectParameter("ingestedTimestamp", ingestedTimestamp) :
+ new ObjectParameter("ingestedTimestamp", typeof(System.DateTime));
+
+
+ var urlParameter = url != null ?
+ new ObjectParameter("url", url) :
+ new ObjectParameter("url", typeof(string));
+
+
+ var titleParameter = title != null ?
+ new ObjectParameter("title", title) :
+ new ObjectParameter("title", typeof(string));
+
+
+ var mediaPreviewUrlParameter = mediaPreviewUrl != null ?
+ new ObjectParameter("mediaPreviewUrl", mediaPreviewUrl) :
+ new ObjectParameter("mediaPreviewUrl", typeof(string));
+
+
+ return ((IObjectContextAdapter)this).ObjectContext.ExecuteFunction("InsertPost", newDocIdParameter, contentParameter, scoreParameter, controversialityParameter, gildedParameter, authorParameter, subredditParameter, publishedTimestampParameter, publishedMonthPrecisionParameter, publishedWeekPrecisionParameter, publishedDayPrecisionParameter, publishedHourPrecisionParameter, publishedMinutePrecisionParameter, ingestedTimestampParameter, urlParameter, titleParameter, mediaPreviewUrlParameter);
+ }
+
+}
+
+}
+
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.tt b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.tt
new file mode 100644
index 000000000..ca8128773
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Context.tt
@@ -0,0 +1,636 @@
+<#@ template language="C#" debug="false" hostspecific="true"#>
+<#@ include file="EF6.Utility.CS.ttinclude"#><#@
+ output extension=".cs"#><#
+
+const string inputFile = @"Reddit.edmx";
+var textTransform = DynamicTextTransformation.Create(this);
+var code = new CodeGenerationTools(this);
+var ef = new MetadataTools(this);
+var typeMapper = new TypeMapper(code, ef, textTransform.Errors);
+var loader = new EdmMetadataLoader(textTransform.Host, textTransform.Errors);
+var itemCollection = loader.CreateEdmItemCollection(inputFile);
+var modelNamespace = loader.GetModelNamespace(inputFile);
+var codeStringGenerator = new CodeStringGenerator(code, typeMapper, ef);
+
+var container = itemCollection.OfType().FirstOrDefault();
+if (container == null)
+{
+ return string.Empty;
+}
+#>
+//------------------------------------------------------------------------------
+//
+// <#=CodeGenerationTools.GetResourceString("Template_GeneratedCodeCommentLine1")#>
+//
+// <#=CodeGenerationTools.GetResourceString("Template_GeneratedCodeCommentLine2")#>
+// <#=CodeGenerationTools.GetResourceString("Template_GeneratedCodeCommentLine3")#>
+//
+//------------------------------------------------------------------------------
+
+<#
+
+var codeNamespace = code.VsNamespaceSuggestion();
+if (!String.IsNullOrEmpty(codeNamespace))
+{
+#>
+namespace <#=code.EscapeNamespace(codeNamespace)#>
+{
+<#
+ PushIndent(" ");
+}
+
+#>
+using System;
+using System.Data.Entity;
+using System.Data.Entity.Infrastructure;
+<#
+if (container.FunctionImports.Any())
+{
+#>
+using System.Data.Entity.Core.Objects;
+using System.Linq;
+<#
+}
+#>
+
+<#=Accessibility.ForType(container)#> partial class <#=code.Escape(container)#> : DbContext
+{
+ public <#=code.Escape(container)#>()
+ : base("name=<#=container.Name#>")
+ {
+<#
+if (!loader.IsLazyLoadingEnabled(container))
+{
+#>
+ this.Configuration.LazyLoadingEnabled = false;
+<#
+}
+
+foreach (var entitySet in container.BaseEntitySets.OfType())
+{
+ // Note: the DbSet members are defined below such that the getter and
+ // setter always have the same accessibility as the DbSet definition
+ if (Accessibility.ForReadOnlyProperty(entitySet) != "public")
+ {
+#>
+ <#=codeStringGenerator.DbSetInitializer(entitySet)#>
+<#
+ }
+}
+#>
+ }
+
+ protected override void OnModelCreating(DbModelBuilder modelBuilder)
+ {
+ throw new UnintentionalCodeFirstException();
+ }
+
+<#
+ foreach (var entitySet in container.BaseEntitySets.OfType())
+ {
+#>
+ <#=codeStringGenerator.DbSet(entitySet)#>
+<#
+ }
+
+ foreach (var edmFunction in container.FunctionImports)
+ {
+ WriteFunctionImport(typeMapper, codeStringGenerator, edmFunction, modelNamespace, includeMergeOption: false);
+ }
+#>
+}
+<#
+
+if (!String.IsNullOrEmpty(codeNamespace))
+{
+ PopIndent();
+#>
+}
+<#
+}
+#>
+<#+
+
+private void WriteFunctionImport(TypeMapper typeMapper, CodeStringGenerator codeStringGenerator, EdmFunction edmFunction, string modelNamespace, bool includeMergeOption)
+{
+ if (typeMapper.IsComposable(edmFunction))
+ {
+#>
+
+ [DbFunction("<#=edmFunction.NamespaceName#>", "<#=edmFunction.Name#>")]
+ <#=codeStringGenerator.ComposableFunctionMethod(edmFunction, modelNamespace)#>
+ {
+<#+
+ codeStringGenerator.WriteFunctionParameters(edmFunction, WriteFunctionParameter);
+#>
+ <#=codeStringGenerator.ComposableCreateQuery(edmFunction, modelNamespace)#>
+ }
+<#+
+ }
+ else
+ {
+#>
+
+ <#=codeStringGenerator.FunctionMethod(edmFunction, modelNamespace, includeMergeOption)#>
+ {
+<#+
+ codeStringGenerator.WriteFunctionParameters(edmFunction, WriteFunctionParameter);
+#>
+ <#=codeStringGenerator.ExecuteFunction(edmFunction, modelNamespace, includeMergeOption)#>
+ }
+<#+
+ if (typeMapper.GenerateMergeOptionFunction(edmFunction, includeMergeOption))
+ {
+ WriteFunctionImport(typeMapper, codeStringGenerator, edmFunction, modelNamespace, includeMergeOption: true);
+ }
+ }
+}
+
+public void WriteFunctionParameter(string name, string isNotNull, string notNullInit, string nullInit)
+{
+#>
+ var <#=name#> = <#=isNotNull#> ?
+ <#=notNullInit#> :
+ <#=nullInit#>;
+
+<#+
+}
+
+public const string TemplateId = "CSharp_DbContext_Context_EF6";
+
+public class CodeStringGenerator
+{
+ private readonly CodeGenerationTools _code;
+ private readonly TypeMapper _typeMapper;
+ private readonly MetadataTools _ef;
+
+ public CodeStringGenerator(CodeGenerationTools code, TypeMapper typeMapper, MetadataTools ef)
+ {
+ ArgumentNotNull(code, "code");
+ ArgumentNotNull(typeMapper, "typeMapper");
+ ArgumentNotNull(ef, "ef");
+
+ _code = code;
+ _typeMapper = typeMapper;
+ _ef = ef;
+ }
+
+ public string Property(EdmProperty edmProperty)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1} {2} {{ {3}get; {4}set; }}",
+ Accessibility.ForProperty(edmProperty),
+ _typeMapper.GetTypeName(edmProperty.TypeUsage),
+ _code.Escape(edmProperty),
+ _code.SpaceAfter(Accessibility.ForGetter(edmProperty)),
+ _code.SpaceAfter(Accessibility.ForSetter(edmProperty)));
+ }
+
+ public string NavigationProperty(NavigationProperty navProp)
+ {
+ var endType = _typeMapper.GetTypeName(navProp.ToEndMember.GetEntityType());
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1} {2} {{ {3}get; {4}set; }}",
+ AccessibilityAndVirtual(Accessibility.ForNavigationProperty(navProp)),
+ navProp.ToEndMember.RelationshipMultiplicity == RelationshipMultiplicity.Many ? ("ICollection<" + endType + ">") : endType,
+ _code.Escape(navProp),
+ _code.SpaceAfter(Accessibility.ForGetter(navProp)),
+ _code.SpaceAfter(Accessibility.ForSetter(navProp)));
+ }
+
+ public string AccessibilityAndVirtual(string accessibility)
+ {
+ return accessibility + (accessibility != "private" ? " virtual" : "");
+ }
+
+ public string EntityClassOpening(EntityType entity)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1}partial class {2}{3}",
+ Accessibility.ForType(entity),
+ _code.SpaceAfter(_code.AbstractOption(entity)),
+ _code.Escape(entity),
+ _code.StringBefore(" : ", _typeMapper.GetTypeName(entity.BaseType)));
+ }
+
+ public string EnumOpening(SimpleType enumType)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} enum {1} : {2}",
+ Accessibility.ForType(enumType),
+ _code.Escape(enumType),
+ _code.Escape(_typeMapper.UnderlyingClrType(enumType)));
+ }
+
+ public void WriteFunctionParameters(EdmFunction edmFunction, Action writeParameter)
+ {
+ var parameters = FunctionImportParameter.Create(edmFunction.Parameters, _code, _ef);
+ foreach (var parameter in parameters.Where(p => p.NeedsLocalVariable))
+ {
+ var isNotNull = parameter.IsNullableOfT ? parameter.FunctionParameterName + ".HasValue" : parameter.FunctionParameterName + " != null";
+ var notNullInit = "new ObjectParameter(\"" + parameter.EsqlParameterName + "\", " + parameter.FunctionParameterName + ")";
+ var nullInit = "new ObjectParameter(\"" + parameter.EsqlParameterName + "\", typeof(" + TypeMapper.FixNamespaces(parameter.RawClrTypeName) + "))";
+ writeParameter(parameter.LocalVariableName, isNotNull, notNullInit, nullInit);
+ }
+ }
+
+ public string ComposableFunctionMethod(EdmFunction edmFunction, string modelNamespace)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} IQueryable<{1}> {2}({3})",
+ AccessibilityAndVirtual(Accessibility.ForMethod(edmFunction)),
+ _typeMapper.GetTypeName(_typeMapper.GetReturnType(edmFunction), modelNamespace),
+ _code.Escape(edmFunction),
+ string.Join(", ", parameters.Select(p => TypeMapper.FixNamespaces(p.FunctionParameterType) + " " + p.FunctionParameterName).ToArray()));
+ }
+
+ public string ComposableCreateQuery(EdmFunction edmFunction, string modelNamespace)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "return ((IObjectContextAdapter)this).ObjectContext.CreateQuery<{0}>(\"[{1}].[{2}]({3})\"{4});",
+ _typeMapper.GetTypeName(_typeMapper.GetReturnType(edmFunction), modelNamespace),
+ edmFunction.NamespaceName,
+ edmFunction.Name,
+ string.Join(", ", parameters.Select(p => "@" + p.EsqlParameterName).ToArray()),
+ _code.StringBefore(", ", string.Join(", ", parameters.Select(p => p.ExecuteParameterName).ToArray())));
+ }
+
+ public string FunctionMethod(EdmFunction edmFunction, string modelNamespace, bool includeMergeOption)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+ var returnType = _typeMapper.GetReturnType(edmFunction);
+
+ var paramList = String.Join(", ", parameters.Select(p => TypeMapper.FixNamespaces(p.FunctionParameterType) + " " + p.FunctionParameterName).ToArray());
+ if (includeMergeOption)
+ {
+ paramList = _code.StringAfter(paramList, ", ") + "MergeOption mergeOption";
+ }
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1} {2}({3})",
+ AccessibilityAndVirtual(Accessibility.ForMethod(edmFunction)),
+ returnType == null ? "int" : "ObjectResult<" + _typeMapper.GetTypeName(returnType, modelNamespace) + ">",
+ _code.Escape(edmFunction),
+ paramList);
+ }
+
+ public string ExecuteFunction(EdmFunction edmFunction, string modelNamespace, bool includeMergeOption)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+ var returnType = _typeMapper.GetReturnType(edmFunction);
+
+ var callParams = _code.StringBefore(", ", String.Join(", ", parameters.Select(p => p.ExecuteParameterName).ToArray()));
+ if (includeMergeOption)
+ {
+ callParams = ", mergeOption" + callParams;
+ }
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "return ((IObjectContextAdapter)this).ObjectContext.ExecuteFunction{0}(\"{1}\"{2});",
+ returnType == null ? "" : "<" + _typeMapper.GetTypeName(returnType, modelNamespace) + ">",
+ edmFunction.Name,
+ callParams);
+ }
+
+ public string DbSet(EntitySet entitySet)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} virtual DbSet<{1}> {2} {{ get; set; }}",
+ Accessibility.ForReadOnlyProperty(entitySet),
+ _typeMapper.GetTypeName(entitySet.ElementType),
+ _code.Escape(entitySet));
+ }
+
+ public string DbSetInitializer(EntitySet entitySet)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} = Set<{1}>();",
+ _code.Escape(entitySet),
+ _typeMapper.GetTypeName(entitySet.ElementType));
+ }
+
+ public string UsingDirectives(bool inHeader, bool includeCollections = true)
+ {
+ return inHeader == string.IsNullOrEmpty(_code.VsNamespaceSuggestion())
+ ? string.Format(
+ CultureInfo.InvariantCulture,
+ "{0}using System;{1}" +
+ "{2}",
+ inHeader ? Environment.NewLine : "",
+ includeCollections ? (Environment.NewLine + "using System.Collections.Generic;") : "",
+ inHeader ? "" : Environment.NewLine)
+ : "";
+ }
+}
+
+public class TypeMapper
+{
+ private const string ExternalTypeNameAttributeName = @"http://schemas.microsoft.com/ado/2006/04/codegeneration:ExternalTypeName";
+
+ private readonly System.Collections.IList _errors;
+ private readonly CodeGenerationTools _code;
+ private readonly MetadataTools _ef;
+
+ public static string FixNamespaces(string typeName)
+ {
+ return typeName.Replace("System.Data.Spatial.", "System.Data.Entity.Spatial.");
+ }
+
+ public TypeMapper(CodeGenerationTools code, MetadataTools ef, System.Collections.IList errors)
+ {
+ ArgumentNotNull(code, "code");
+ ArgumentNotNull(ef, "ef");
+ ArgumentNotNull(errors, "errors");
+
+ _code = code;
+ _ef = ef;
+ _errors = errors;
+ }
+
+ public string GetTypeName(TypeUsage typeUsage)
+ {
+ return typeUsage == null ? null : GetTypeName(typeUsage.EdmType, _ef.IsNullable(typeUsage), modelNamespace: null);
+ }
+
+ public string GetTypeName(EdmType edmType)
+ {
+ return GetTypeName(edmType, isNullable: null, modelNamespace: null);
+ }
+
+ public string GetTypeName(TypeUsage typeUsage, string modelNamespace)
+ {
+ return typeUsage == null ? null : GetTypeName(typeUsage.EdmType, _ef.IsNullable(typeUsage), modelNamespace);
+ }
+
+ public string GetTypeName(EdmType edmType, string modelNamespace)
+ {
+ return GetTypeName(edmType, isNullable: null, modelNamespace: modelNamespace);
+ }
+
+ public string GetTypeName(EdmType edmType, bool? isNullable, string modelNamespace)
+ {
+ if (edmType == null)
+ {
+ return null;
+ }
+
+ var collectionType = edmType as CollectionType;
+ if (collectionType != null)
+ {
+ return String.Format(CultureInfo.InvariantCulture, "ICollection<{0}>", GetTypeName(collectionType.TypeUsage, modelNamespace));
+ }
+
+ var typeName = _code.Escape(edmType.MetadataProperties
+ .Where(p => p.Name == ExternalTypeNameAttributeName)
+ .Select(p => (string)p.Value)
+ .FirstOrDefault())
+ ?? (modelNamespace != null && edmType.NamespaceName != modelNamespace ?
+ _code.CreateFullName(_code.EscapeNamespace(edmType.NamespaceName), _code.Escape(edmType)) :
+ _code.Escape(edmType));
+
+ if (edmType is StructuralType)
+ {
+ return typeName;
+ }
+
+ if (edmType is SimpleType)
+ {
+ var clrType = UnderlyingClrType(edmType);
+ if (!IsEnumType(edmType))
+ {
+ typeName = _code.Escape(clrType);
+ }
+
+ typeName = FixNamespaces(typeName);
+
+ return clrType.IsValueType && isNullable == true ?
+ String.Format(CultureInfo.InvariantCulture, "Nullable<{0}>", typeName) :
+ typeName;
+ }
+
+ throw new ArgumentException("edmType");
+ }
+
+ public Type UnderlyingClrType(EdmType edmType)
+ {
+ ArgumentNotNull(edmType, "edmType");
+
+ var primitiveType = edmType as PrimitiveType;
+ if (primitiveType != null)
+ {
+ return primitiveType.ClrEquivalentType;
+ }
+
+ if (IsEnumType(edmType))
+ {
+ return GetEnumUnderlyingType(edmType).ClrEquivalentType;
+ }
+
+ return typeof(object);
+ }
+
+ public object GetEnumMemberValue(MetadataItem enumMember)
+ {
+ ArgumentNotNull(enumMember, "enumMember");
+
+ var valueProperty = enumMember.GetType().GetProperty("Value");
+ return valueProperty == null ? null : valueProperty.GetValue(enumMember, null);
+ }
+
+ public string GetEnumMemberName(MetadataItem enumMember)
+ {
+ ArgumentNotNull(enumMember, "enumMember");
+
+ var nameProperty = enumMember.GetType().GetProperty("Name");
+ return nameProperty == null ? null : (string)nameProperty.GetValue(enumMember, null);
+ }
+
+ public System.Collections.IEnumerable GetEnumMembers(EdmType enumType)
+ {
+ ArgumentNotNull(enumType, "enumType");
+
+ var membersProperty = enumType.GetType().GetProperty("Members");
+ return membersProperty != null
+ ? (System.Collections.IEnumerable)membersProperty.GetValue(enumType, null)
+ : Enumerable.Empty();
+ }
+
+ public bool EnumIsFlags(EdmType enumType)
+ {
+ ArgumentNotNull(enumType, "enumType");
+
+ var isFlagsProperty = enumType.GetType().GetProperty("IsFlags");
+ return isFlagsProperty != null && (bool)isFlagsProperty.GetValue(enumType, null);
+ }
+
+ public bool IsEnumType(GlobalItem edmType)
+ {
+ ArgumentNotNull(edmType, "edmType");
+
+ return edmType.GetType().Name == "EnumType";
+ }
+
+ public PrimitiveType GetEnumUnderlyingType(EdmType enumType)
+ {
+ ArgumentNotNull(enumType, "enumType");
+
+ return (PrimitiveType)enumType.GetType().GetProperty("UnderlyingType").GetValue(enumType, null);
+ }
+
+ public string CreateLiteral(object value)
+ {
+ if (value == null || value.GetType() != typeof(TimeSpan))
+ {
+ return _code.CreateLiteral(value);
+ }
+
+ return string.Format(CultureInfo.InvariantCulture, "new TimeSpan({0})", ((TimeSpan)value).Ticks);
+ }
+
+ public bool VerifyCaseInsensitiveTypeUniqueness(IEnumerable types, string sourceFile)
+ {
+ ArgumentNotNull(types, "types");
+ ArgumentNotNull(sourceFile, "sourceFile");
+
+ var hash = new HashSet(StringComparer.InvariantCultureIgnoreCase);
+ if (types.Any(item => !hash.Add(item)))
+ {
+ _errors.Add(
+ new CompilerError(sourceFile, -1, -1, "6023",
+ String.Format(CultureInfo.CurrentCulture, CodeGenerationTools.GetResourceString("Template_CaseInsensitiveTypeConflict"))));
+ return false;
+ }
+ return true;
+ }
+
+ public IEnumerable GetEnumItemsToGenerate(IEnumerable itemCollection)
+ {
+ return GetItemsToGenerate(itemCollection)
+ .Where(e => IsEnumType(e));
+ }
+
+ public IEnumerable GetItemsToGenerate(IEnumerable itemCollection) where T: EdmType
+ {
+ return itemCollection
+ .OfType()
+ .Where(i => !i.MetadataProperties.Any(p => p.Name == ExternalTypeNameAttributeName))
+ .OrderBy(i => i.Name);
+ }
+
+ public IEnumerable GetAllGlobalItems(IEnumerable itemCollection)
+ {
+ return itemCollection
+ .Where(i => i is EntityType || i is ComplexType || i is EntityContainer || IsEnumType(i))
+ .Select(g => GetGlobalItemName(g));
+ }
+
+ public string GetGlobalItemName(GlobalItem item)
+ {
+ if (item is EdmType)
+ {
+ return ((EdmType)item).Name;
+ }
+ else
+ {
+ return ((EntityContainer)item).Name;
+ }
+ }
+
+ public IEnumerable GetSimpleProperties(EntityType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetSimpleProperties(ComplexType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetComplexProperties(EntityType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is ComplexType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetComplexProperties(ComplexType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is ComplexType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetPropertiesWithDefaultValues(EntityType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type && p.DefaultValue != null);
+ }
+
+ public IEnumerable GetPropertiesWithDefaultValues(ComplexType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type && p.DefaultValue != null);
+ }
+
+ public IEnumerable GetNavigationProperties(EntityType type)
+ {
+ return type.NavigationProperties.Where(np => np.DeclaringType == type);
+ }
+
+ public IEnumerable GetCollectionNavigationProperties(EntityType type)
+ {
+ return type.NavigationProperties.Where(np => np.DeclaringType == type && np.ToEndMember.RelationshipMultiplicity == RelationshipMultiplicity.Many);
+ }
+
+ public FunctionParameter GetReturnParameter(EdmFunction edmFunction)
+ {
+ ArgumentNotNull(edmFunction, "edmFunction");
+
+ var returnParamsProperty = edmFunction.GetType().GetProperty("ReturnParameters");
+ return returnParamsProperty == null
+ ? edmFunction.ReturnParameter
+ : ((IEnumerable)returnParamsProperty.GetValue(edmFunction, null)).FirstOrDefault();
+ }
+
+ public bool IsComposable(EdmFunction edmFunction)
+ {
+ ArgumentNotNull(edmFunction, "edmFunction");
+
+ var isComposableProperty = edmFunction.GetType().GetProperty("IsComposableAttribute");
+ return isComposableProperty != null && (bool)isComposableProperty.GetValue(edmFunction, null);
+ }
+
+ public IEnumerable GetParameters(EdmFunction edmFunction)
+ {
+ return FunctionImportParameter.Create(edmFunction.Parameters, _code, _ef);
+ }
+
+ public TypeUsage GetReturnType(EdmFunction edmFunction)
+ {
+ var returnParam = GetReturnParameter(edmFunction);
+ return returnParam == null ? null : _ef.GetElementType(returnParam.TypeUsage);
+ }
+
+ public bool GenerateMergeOptionFunction(EdmFunction edmFunction, bool includeMergeOption)
+ {
+ var returnType = GetReturnType(edmFunction);
+ return !includeMergeOption && returnType != null && returnType.EdmType.BuiltInTypeKind == BuiltInTypeKind.EntityType;
+ }
+}
+
+public static void ArgumentNotNull(T arg, string name) where T : class
+{
+ if (arg == null)
+ {
+ throw new ArgumentNullException(name);
+ }
+}
+#>
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Designer.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Designer.cs
new file mode 100644
index 000000000..354ffe52a
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.Designer.cs
@@ -0,0 +1,10 @@
+// T4 code generation is enabled for model 'C:\Users\jomclean\Source\RedditTemplate\Src\RedditCore\DataModel\Reddit.edmx'.
+// To enable legacy code generation, change the value of the 'Code Generation Strategy' designer
+// property to 'Legacy ObjectContext'. This property is available in the Properties Window when the model
+// is open in the designer.
+
+// If no context and entity classes have been generated, it may be because you created an empty model but
+// have not yet chosen which version of Entity Framework to use. To generate a context class and entity
+// classes for your model, open the model in the designer, right-click on the designer surface, and
+// select 'Update Model from Database...', 'Generate Database from Model...', or 'Add Code Generation
+// Item...'.
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.cs
new file mode 100644
index 000000000..c36263efa
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.cs
@@ -0,0 +1,12 @@
+
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.edmx b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.edmx
new file mode 100644
index 000000000..ddc8f2db4
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.edmx
@@ -0,0 +1,571 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ SELECT
+ [UserDefinedEntityDefinitions].[regex] AS [regex],
+ [UserDefinedEntityDefinitions].[entityType] AS [entityType],
+ [UserDefinedEntityDefinitions].[entityValue] AS [entityValue],
+ [UserDefinedEntityDefinitions].[color] AS [color]
+ FROM [reddit].[UserDefinedEntityDefinitions] AS [UserDefinedEntityDefinitions]
+
+
+ SELECT
+ [AllCommentsView].[id] AS [id],
+ [AllCommentsView].[content] AS [content],
+ [AllCommentsView].[score] AS [score],
+ [AllCommentsView].[controversiality] AS [controversiality],
+ [AllCommentsView].[gilded] AS [gilded],
+ [AllCommentsView].[author] AS [author],
+ [AllCommentsView].[subreddit] AS [subreddit],
+ [AllCommentsView].[isComment] AS [isComment],
+ [AllCommentsView].[publishedTimestamp] AS [publishedTimestamp],
+ [AllCommentsView].[publishedMonthPrecision] AS [publishedMonthPrecision],
+ [AllCommentsView].[publishedWeekPrecision] AS [publishedWeekPrecision],
+ [AllCommentsView].[publishedDayPrecision] AS [publishedDayPrecision],
+ [AllCommentsView].[publishedHourPrecision] AS [publishedHourPrecision],
+ [AllCommentsView].[publishedMinutePrecision] AS [publishedMinutePrecision],
+ [AllCommentsView].[ingestedTimestamp] AS [ingestedTimestamp],
+ [AllCommentsView].[url] AS [url],
+ [AllCommentsView].[sentiment] AS [sentiment],
+ [AllCommentsView].[parentId] AS [parentId],
+ [AllCommentsView].[postId] AS [postId],
+ [AllCommentsView].[parentUrl] AS [parentUrl],
+ [AllCommentsView].[postUrl] AS [postUrl],
+ [AllCommentsView].[documentContainsUserDefinedEntities] AS [documentContainsUserDefinedEntities]
+ FROM [reddit].[AllCommentsView] AS [AllCommentsView]
+
+
+ SELECT
+ [AllPostsView].[id] AS [id],
+ [AllPostsView].[content] AS [content],
+ [AllPostsView].[score] AS [score],
+ [AllPostsView].[controversiality] AS [controversiality],
+ [AllPostsView].[gilded] AS [gilded],
+ [AllPostsView].[author] AS [author],
+ [AllPostsView].[subreddit] AS [subreddit],
+ [AllPostsView].[isComment] AS [isComment],
+ [AllPostsView].[publishedTimestamp] AS [publishedTimestamp],
+ [AllPostsView].[publishedMonthPrecision] AS [publishedMonthPrecision],
+ [AllPostsView].[publishedWeekPrecision] AS [publishedWeekPrecision],
+ [AllPostsView].[publishedDayPrecision] AS [publishedDayPrecision],
+ [AllPostsView].[publishedHourPrecision] AS [publishedHourPrecision],
+ [AllPostsView].[publishedMinutePrecision] AS [publishedMinutePrecision],
+ [AllPostsView].[ingestedTimestamp] AS [ingestedTimestamp],
+ [AllPostsView].[url] AS [url],
+ [AllPostsView].[sentiment] AS [sentiment],
+ [AllPostsView].[title] AS [title],
+ [AllPostsView].[mediaPreviewUrl] AS [mediaPreviewUrl],
+ [AllPostsView].[documentContainsUserDefinedEntities] AS [documentContainsUserDefinedEntities]
+ FROM [reddit].[AllPostsView] AS [AllPostsView]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.edmx.diagram b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.edmx.diagram
new file mode 100644
index 000000000..2d6f93344
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.edmx.diagram
@@ -0,0 +1,17 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.tt b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.tt
new file mode 100644
index 000000000..eeec21330
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Reddit.tt
@@ -0,0 +1,733 @@
+<#@ template language="C#" debug="false" hostspecific="true"#>
+<#@ include file="EF6.Utility.CS.ttinclude"#><#@
+ output extension=".cs"#><#
+
+const string inputFile = @"Reddit.edmx";
+var textTransform = DynamicTextTransformation.Create(this);
+var code = new CodeGenerationTools(this);
+var ef = new MetadataTools(this);
+var typeMapper = new TypeMapper(code, ef, textTransform.Errors);
+var fileManager = EntityFrameworkTemplateFileManager.Create(this);
+var itemCollection = new EdmMetadataLoader(textTransform.Host, textTransform.Errors).CreateEdmItemCollection(inputFile);
+var codeStringGenerator = new CodeStringGenerator(code, typeMapper, ef);
+
+if (!typeMapper.VerifyCaseInsensitiveTypeUniqueness(typeMapper.GetAllGlobalItems(itemCollection), inputFile))
+{
+ return string.Empty;
+}
+
+WriteHeader(codeStringGenerator, fileManager);
+
+foreach (var entity in typeMapper.GetItemsToGenerate(itemCollection))
+{
+ fileManager.StartNewFile(entity.Name + ".cs");
+ BeginNamespace(code);
+#>
+<#=codeStringGenerator.UsingDirectives(inHeader: false)#>
+<#=codeStringGenerator.EntityClassOpening(entity)#>
+{
+<#
+ var propertiesWithDefaultValues = typeMapper.GetPropertiesWithDefaultValues(entity);
+ var collectionNavigationProperties = typeMapper.GetCollectionNavigationProperties(entity);
+ var complexProperties = typeMapper.GetComplexProperties(entity);
+
+ if (propertiesWithDefaultValues.Any() || collectionNavigationProperties.Any() || complexProperties.Any())
+ {
+#>
+ [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors")]
+ public <#=code.Escape(entity)#>()
+ {
+<#
+ foreach (var edmProperty in propertiesWithDefaultValues)
+ {
+#>
+ this.<#=code.Escape(edmProperty)#> = <#=typeMapper.CreateLiteral(edmProperty.DefaultValue)#>;
+<#
+ }
+
+ foreach (var navigationProperty in collectionNavigationProperties)
+ {
+#>
+ this.<#=code.Escape(navigationProperty)#> = new HashSet<<#=typeMapper.GetTypeName(navigationProperty.ToEndMember.GetEntityType())#>>();
+<#
+ }
+
+ foreach (var complexProperty in complexProperties)
+ {
+#>
+ this.<#=code.Escape(complexProperty)#> = new <#=typeMapper.GetTypeName(complexProperty.TypeUsage)#>();
+<#
+ }
+#>
+ }
+
+<#
+ }
+
+ var simpleProperties = typeMapper.GetSimpleProperties(entity);
+ if (simpleProperties.Any())
+ {
+ foreach (var edmProperty in simpleProperties)
+ {
+#>
+ <#=codeStringGenerator.Property(edmProperty)#>
+<#
+ }
+ }
+
+ if (complexProperties.Any())
+ {
+#>
+
+<#
+ foreach(var complexProperty in complexProperties)
+ {
+#>
+ <#=codeStringGenerator.Property(complexProperty)#>
+<#
+ }
+ }
+
+ var navigationProperties = typeMapper.GetNavigationProperties(entity);
+ if (navigationProperties.Any())
+ {
+#>
+
+<#
+ foreach (var navigationProperty in navigationProperties)
+ {
+ if (navigationProperty.ToEndMember.RelationshipMultiplicity == RelationshipMultiplicity.Many)
+ {
+#>
+ [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
+<#
+ }
+#>
+ <#=codeStringGenerator.NavigationProperty(navigationProperty)#>
+<#
+ }
+ }
+#>
+}
+<#
+ EndNamespace(code);
+}
+
+foreach (var complex in typeMapper.GetItemsToGenerate(itemCollection))
+{
+ fileManager.StartNewFile(complex.Name + ".cs");
+ BeginNamespace(code);
+#>
+<#=codeStringGenerator.UsingDirectives(inHeader: false, includeCollections: false)#>
+<#=Accessibility.ForType(complex)#> partial class <#=code.Escape(complex)#>
+{
+<#
+ var complexProperties = typeMapper.GetComplexProperties(complex);
+ var propertiesWithDefaultValues = typeMapper.GetPropertiesWithDefaultValues(complex);
+
+ if (propertiesWithDefaultValues.Any() || complexProperties.Any())
+ {
+#>
+ public <#=code.Escape(complex)#>()
+ {
+<#
+ foreach (var edmProperty in propertiesWithDefaultValues)
+ {
+#>
+ this.<#=code.Escape(edmProperty)#> = <#=typeMapper.CreateLiteral(edmProperty.DefaultValue)#>;
+<#
+ }
+
+ foreach (var complexProperty in complexProperties)
+ {
+#>
+ this.<#=code.Escape(complexProperty)#> = new <#=typeMapper.GetTypeName(complexProperty.TypeUsage)#>();
+<#
+ }
+#>
+ }
+
+<#
+ }
+
+ var simpleProperties = typeMapper.GetSimpleProperties(complex);
+ if (simpleProperties.Any())
+ {
+ foreach(var edmProperty in simpleProperties)
+ {
+#>
+ <#=codeStringGenerator.Property(edmProperty)#>
+<#
+ }
+ }
+
+ if (complexProperties.Any())
+ {
+#>
+
+<#
+ foreach(var edmProperty in complexProperties)
+ {
+#>
+ <#=codeStringGenerator.Property(edmProperty)#>
+<#
+ }
+ }
+#>
+}
+<#
+ EndNamespace(code);
+}
+
+foreach (var enumType in typeMapper.GetEnumItemsToGenerate(itemCollection))
+{
+ fileManager.StartNewFile(enumType.Name + ".cs");
+ BeginNamespace(code);
+#>
+<#=codeStringGenerator.UsingDirectives(inHeader: false, includeCollections: false)#>
+<#
+ if (typeMapper.EnumIsFlags(enumType))
+ {
+#>
+[Flags]
+<#
+ }
+#>
+<#=codeStringGenerator.EnumOpening(enumType)#>
+{
+<#
+ var foundOne = false;
+
+ foreach (MetadataItem member in typeMapper.GetEnumMembers(enumType))
+ {
+ foundOne = true;
+#>
+ <#=code.Escape(typeMapper.GetEnumMemberName(member))#> = <#=typeMapper.GetEnumMemberValue(member)#>,
+<#
+ }
+
+ if (foundOne)
+ {
+ this.GenerationEnvironment.Remove(this.GenerationEnvironment.Length - 3, 1);
+ }
+#>
+}
+<#
+ EndNamespace(code);
+}
+
+fileManager.Process();
+
+#>
+<#+
+
+public void WriteHeader(CodeStringGenerator codeStringGenerator, EntityFrameworkTemplateFileManager fileManager)
+{
+ fileManager.StartHeader();
+#>
+//------------------------------------------------------------------------------
+//
+// <#=CodeGenerationTools.GetResourceString("Template_GeneratedCodeCommentLine1")#>
+//
+// <#=CodeGenerationTools.GetResourceString("Template_GeneratedCodeCommentLine2")#>
+// <#=CodeGenerationTools.GetResourceString("Template_GeneratedCodeCommentLine3")#>
+//
+//------------------------------------------------------------------------------
+<#=codeStringGenerator.UsingDirectives(inHeader: true)#>
+<#+
+ fileManager.EndBlock();
+}
+
+public void BeginNamespace(CodeGenerationTools code)
+{
+ var codeNamespace = code.VsNamespaceSuggestion();
+ if (!String.IsNullOrEmpty(codeNamespace))
+ {
+#>
+namespace <#=code.EscapeNamespace(codeNamespace)#>
+{
+<#+
+ PushIndent(" ");
+ }
+}
+
+public void EndNamespace(CodeGenerationTools code)
+{
+ if (!String.IsNullOrEmpty(code.VsNamespaceSuggestion()))
+ {
+ PopIndent();
+#>
+}
+<#+
+ }
+}
+
+public const string TemplateId = "CSharp_DbContext_Types_EF6";
+
+public class CodeStringGenerator
+{
+ private readonly CodeGenerationTools _code;
+ private readonly TypeMapper _typeMapper;
+ private readonly MetadataTools _ef;
+
+ public CodeStringGenerator(CodeGenerationTools code, TypeMapper typeMapper, MetadataTools ef)
+ {
+ ArgumentNotNull(code, "code");
+ ArgumentNotNull(typeMapper, "typeMapper");
+ ArgumentNotNull(ef, "ef");
+
+ _code = code;
+ _typeMapper = typeMapper;
+ _ef = ef;
+ }
+
+ public string Property(EdmProperty edmProperty)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1} {2} {{ {3}get; {4}set; }}",
+ Accessibility.ForProperty(edmProperty),
+ _typeMapper.GetTypeName(edmProperty.TypeUsage),
+ _code.Escape(edmProperty),
+ _code.SpaceAfter(Accessibility.ForGetter(edmProperty)),
+ _code.SpaceAfter(Accessibility.ForSetter(edmProperty)));
+ }
+
+ public string NavigationProperty(NavigationProperty navProp)
+ {
+ var endType = _typeMapper.GetTypeName(navProp.ToEndMember.GetEntityType());
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1} {2} {{ {3}get; {4}set; }}",
+ AccessibilityAndVirtual(Accessibility.ForNavigationProperty(navProp)),
+ navProp.ToEndMember.RelationshipMultiplicity == RelationshipMultiplicity.Many ? ("ICollection<" + endType + ">") : endType,
+ _code.Escape(navProp),
+ _code.SpaceAfter(Accessibility.ForGetter(navProp)),
+ _code.SpaceAfter(Accessibility.ForSetter(navProp)));
+ }
+
+ public string AccessibilityAndVirtual(string accessibility)
+ {
+ return accessibility + (accessibility != "private" ? " virtual" : "");
+ }
+
+ public string EntityClassOpening(EntityType entity)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1}partial class {2}{3}",
+ Accessibility.ForType(entity),
+ _code.SpaceAfter(_code.AbstractOption(entity)),
+ _code.Escape(entity),
+ _code.StringBefore(" : ", _typeMapper.GetTypeName(entity.BaseType)));
+ }
+
+ public string EnumOpening(SimpleType enumType)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} enum {1} : {2}",
+ Accessibility.ForType(enumType),
+ _code.Escape(enumType),
+ _code.Escape(_typeMapper.UnderlyingClrType(enumType)));
+ }
+
+ public void WriteFunctionParameters(EdmFunction edmFunction, Action writeParameter)
+ {
+ var parameters = FunctionImportParameter.Create(edmFunction.Parameters, _code, _ef);
+ foreach (var parameter in parameters.Where(p => p.NeedsLocalVariable))
+ {
+ var isNotNull = parameter.IsNullableOfT ? parameter.FunctionParameterName + ".HasValue" : parameter.FunctionParameterName + " != null";
+ var notNullInit = "new ObjectParameter(\"" + parameter.EsqlParameterName + "\", " + parameter.FunctionParameterName + ")";
+ var nullInit = "new ObjectParameter(\"" + parameter.EsqlParameterName + "\", typeof(" + TypeMapper.FixNamespaces(parameter.RawClrTypeName) + "))";
+ writeParameter(parameter.LocalVariableName, isNotNull, notNullInit, nullInit);
+ }
+ }
+
+ public string ComposableFunctionMethod(EdmFunction edmFunction, string modelNamespace)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} IQueryable<{1}> {2}({3})",
+ AccessibilityAndVirtual(Accessibility.ForMethod(edmFunction)),
+ _typeMapper.GetTypeName(_typeMapper.GetReturnType(edmFunction), modelNamespace),
+ _code.Escape(edmFunction),
+ string.Join(", ", parameters.Select(p => TypeMapper.FixNamespaces(p.FunctionParameterType) + " " + p.FunctionParameterName).ToArray()));
+ }
+
+ public string ComposableCreateQuery(EdmFunction edmFunction, string modelNamespace)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "return ((IObjectContextAdapter)this).ObjectContext.CreateQuery<{0}>(\"[{1}].[{2}]({3})\"{4});",
+ _typeMapper.GetTypeName(_typeMapper.GetReturnType(edmFunction), modelNamespace),
+ edmFunction.NamespaceName,
+ edmFunction.Name,
+ string.Join(", ", parameters.Select(p => "@" + p.EsqlParameterName).ToArray()),
+ _code.StringBefore(", ", string.Join(", ", parameters.Select(p => p.ExecuteParameterName).ToArray())));
+ }
+
+ public string FunctionMethod(EdmFunction edmFunction, string modelNamespace, bool includeMergeOption)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+ var returnType = _typeMapper.GetReturnType(edmFunction);
+
+ var paramList = String.Join(", ", parameters.Select(p => TypeMapper.FixNamespaces(p.FunctionParameterType) + " " + p.FunctionParameterName).ToArray());
+ if (includeMergeOption)
+ {
+ paramList = _code.StringAfter(paramList, ", ") + "MergeOption mergeOption";
+ }
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} {1} {2}({3})",
+ AccessibilityAndVirtual(Accessibility.ForMethod(edmFunction)),
+ returnType == null ? "int" : "ObjectResult<" + _typeMapper.GetTypeName(returnType, modelNamespace) + ">",
+ _code.Escape(edmFunction),
+ paramList);
+ }
+
+ public string ExecuteFunction(EdmFunction edmFunction, string modelNamespace, bool includeMergeOption)
+ {
+ var parameters = _typeMapper.GetParameters(edmFunction);
+ var returnType = _typeMapper.GetReturnType(edmFunction);
+
+ var callParams = _code.StringBefore(", ", String.Join(", ", parameters.Select(p => p.ExecuteParameterName).ToArray()));
+ if (includeMergeOption)
+ {
+ callParams = ", mergeOption" + callParams;
+ }
+
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "return ((IObjectContextAdapter)this).ObjectContext.ExecuteFunction{0}(\"{1}\"{2});",
+ returnType == null ? "" : "<" + _typeMapper.GetTypeName(returnType, modelNamespace) + ">",
+ edmFunction.Name,
+ callParams);
+ }
+
+ public string DbSet(EntitySet entitySet)
+ {
+ return string.Format(
+ CultureInfo.InvariantCulture,
+ "{0} virtual DbSet<{1}> {2} {{ get; set; }}",
+ Accessibility.ForReadOnlyProperty(entitySet),
+ _typeMapper.GetTypeName(entitySet.ElementType),
+ _code.Escape(entitySet));
+ }
+
+ public string UsingDirectives(bool inHeader, bool includeCollections = true)
+ {
+ return inHeader == string.IsNullOrEmpty(_code.VsNamespaceSuggestion())
+ ? string.Format(
+ CultureInfo.InvariantCulture,
+ "{0}using System;{1}" +
+ "{2}",
+ inHeader ? Environment.NewLine : "",
+ includeCollections ? (Environment.NewLine + "using System.Collections.Generic;") : "",
+ inHeader ? "" : Environment.NewLine)
+ : "";
+ }
+}
+
+public class TypeMapper
+{
+ private const string ExternalTypeNameAttributeName = @"http://schemas.microsoft.com/ado/2006/04/codegeneration:ExternalTypeName";
+
+ private readonly System.Collections.IList _errors;
+ private readonly CodeGenerationTools _code;
+ private readonly MetadataTools _ef;
+
+ public TypeMapper(CodeGenerationTools code, MetadataTools ef, System.Collections.IList errors)
+ {
+ ArgumentNotNull(code, "code");
+ ArgumentNotNull(ef, "ef");
+ ArgumentNotNull(errors, "errors");
+
+ _code = code;
+ _ef = ef;
+ _errors = errors;
+ }
+
+ public static string FixNamespaces(string typeName)
+ {
+ return typeName.Replace("System.Data.Spatial.", "System.Data.Entity.Spatial.");
+ }
+
+ public string GetTypeName(TypeUsage typeUsage)
+ {
+ return typeUsage == null ? null : GetTypeName(typeUsage.EdmType, _ef.IsNullable(typeUsage), modelNamespace: null);
+ }
+
+ public string GetTypeName(EdmType edmType)
+ {
+ return GetTypeName(edmType, isNullable: null, modelNamespace: null);
+ }
+
+ public string GetTypeName(TypeUsage typeUsage, string modelNamespace)
+ {
+ return typeUsage == null ? null : GetTypeName(typeUsage.EdmType, _ef.IsNullable(typeUsage), modelNamespace);
+ }
+
+ public string GetTypeName(EdmType edmType, string modelNamespace)
+ {
+ return GetTypeName(edmType, isNullable: null, modelNamespace: modelNamespace);
+ }
+
+ public string GetTypeName(EdmType edmType, bool? isNullable, string modelNamespace)
+ {
+ if (edmType == null)
+ {
+ return null;
+ }
+
+ var collectionType = edmType as CollectionType;
+ if (collectionType != null)
+ {
+ return String.Format(CultureInfo.InvariantCulture, "ICollection<{0}>", GetTypeName(collectionType.TypeUsage, modelNamespace));
+ }
+
+ var typeName = _code.Escape(edmType.MetadataProperties
+ .Where(p => p.Name == ExternalTypeNameAttributeName)
+ .Select(p => (string)p.Value)
+ .FirstOrDefault())
+ ?? (modelNamespace != null && edmType.NamespaceName != modelNamespace ?
+ _code.CreateFullName(_code.EscapeNamespace(edmType.NamespaceName), _code.Escape(edmType)) :
+ _code.Escape(edmType));
+
+ if (edmType is StructuralType)
+ {
+ return typeName;
+ }
+
+ if (edmType is SimpleType)
+ {
+ var clrType = UnderlyingClrType(edmType);
+ if (!IsEnumType(edmType))
+ {
+ typeName = _code.Escape(clrType);
+ }
+
+ typeName = FixNamespaces(typeName);
+
+ return clrType.IsValueType && isNullable == true ?
+ String.Format(CultureInfo.InvariantCulture, "Nullable<{0}>", typeName) :
+ typeName;
+ }
+
+ throw new ArgumentException("edmType");
+ }
+
+ public Type UnderlyingClrType(EdmType edmType)
+ {
+ ArgumentNotNull(edmType, "edmType");
+
+ var primitiveType = edmType as PrimitiveType;
+ if (primitiveType != null)
+ {
+ return primitiveType.ClrEquivalentType;
+ }
+
+ if (IsEnumType(edmType))
+ {
+ return GetEnumUnderlyingType(edmType).ClrEquivalentType;
+ }
+
+ return typeof(object);
+ }
+
+ public object GetEnumMemberValue(MetadataItem enumMember)
+ {
+ ArgumentNotNull(enumMember, "enumMember");
+
+ var valueProperty = enumMember.GetType().GetProperty("Value");
+ return valueProperty == null ? null : valueProperty.GetValue(enumMember, null);
+ }
+
+ public string GetEnumMemberName(MetadataItem enumMember)
+ {
+ ArgumentNotNull(enumMember, "enumMember");
+
+ var nameProperty = enumMember.GetType().GetProperty("Name");
+ return nameProperty == null ? null : (string)nameProperty.GetValue(enumMember, null);
+ }
+
+ public System.Collections.IEnumerable GetEnumMembers(EdmType enumType)
+ {
+ ArgumentNotNull(enumType, "enumType");
+
+ var membersProperty = enumType.GetType().GetProperty("Members");
+ return membersProperty != null
+ ? (System.Collections.IEnumerable)membersProperty.GetValue(enumType, null)
+ : Enumerable.Empty();
+ }
+
+ public bool EnumIsFlags(EdmType enumType)
+ {
+ ArgumentNotNull(enumType, "enumType");
+
+ var isFlagsProperty = enumType.GetType().GetProperty("IsFlags");
+ return isFlagsProperty != null && (bool)isFlagsProperty.GetValue(enumType, null);
+ }
+
+ public bool IsEnumType(GlobalItem edmType)
+ {
+ ArgumentNotNull(edmType, "edmType");
+
+ return edmType.GetType().Name == "EnumType";
+ }
+
+ public PrimitiveType GetEnumUnderlyingType(EdmType enumType)
+ {
+ ArgumentNotNull(enumType, "enumType");
+
+ return (PrimitiveType)enumType.GetType().GetProperty("UnderlyingType").GetValue(enumType, null);
+ }
+
+ public string CreateLiteral(object value)
+ {
+ if (value == null || value.GetType() != typeof(TimeSpan))
+ {
+ return _code.CreateLiteral(value);
+ }
+
+ return string.Format(CultureInfo.InvariantCulture, "new TimeSpan({0})", ((TimeSpan)value).Ticks);
+ }
+
+ public bool VerifyCaseInsensitiveTypeUniqueness(IEnumerable types, string sourceFile)
+ {
+ ArgumentNotNull(types, "types");
+ ArgumentNotNull(sourceFile, "sourceFile");
+
+ var hash = new HashSet(StringComparer.InvariantCultureIgnoreCase);
+ if (types.Any(item => !hash.Add(item)))
+ {
+ _errors.Add(
+ new CompilerError(sourceFile, -1, -1, "6023",
+ String.Format(CultureInfo.CurrentCulture, CodeGenerationTools.GetResourceString("Template_CaseInsensitiveTypeConflict"))));
+ return false;
+ }
+ return true;
+ }
+
+ public IEnumerable GetEnumItemsToGenerate(IEnumerable itemCollection)
+ {
+ return GetItemsToGenerate(itemCollection)
+ .Where(e => IsEnumType(e));
+ }
+
+ public IEnumerable GetItemsToGenerate(IEnumerable itemCollection) where T: EdmType
+ {
+ return itemCollection
+ .OfType()
+ .Where(i => !i.MetadataProperties.Any(p => p.Name == ExternalTypeNameAttributeName))
+ .OrderBy(i => i.Name);
+ }
+
+ public IEnumerable GetAllGlobalItems(IEnumerable itemCollection)
+ {
+ return itemCollection
+ .Where(i => i is EntityType || i is ComplexType || i is EntityContainer || IsEnumType(i))
+ .Select(g => GetGlobalItemName(g));
+ }
+
+ public string GetGlobalItemName(GlobalItem item)
+ {
+ if (item is EdmType)
+ {
+ return ((EdmType)item).Name;
+ }
+ else
+ {
+ return ((EntityContainer)item).Name;
+ }
+ }
+
+ public IEnumerable GetSimpleProperties(EntityType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetSimpleProperties(ComplexType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetComplexProperties(EntityType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is ComplexType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetComplexProperties(ComplexType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is ComplexType && p.DeclaringType == type);
+ }
+
+ public IEnumerable GetPropertiesWithDefaultValues(EntityType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type && p.DefaultValue != null);
+ }
+
+ public IEnumerable GetPropertiesWithDefaultValues(ComplexType type)
+ {
+ return type.Properties.Where(p => p.TypeUsage.EdmType is SimpleType && p.DeclaringType == type && p.DefaultValue != null);
+ }
+
+ public IEnumerable GetNavigationProperties(EntityType type)
+ {
+ return type.NavigationProperties.Where(np => np.DeclaringType == type);
+ }
+
+ public IEnumerable GetCollectionNavigationProperties(EntityType type)
+ {
+ return type.NavigationProperties.Where(np => np.DeclaringType == type && np.ToEndMember.RelationshipMultiplicity == RelationshipMultiplicity.Many);
+ }
+
+ public FunctionParameter GetReturnParameter(EdmFunction edmFunction)
+ {
+ ArgumentNotNull(edmFunction, "edmFunction");
+
+ var returnParamsProperty = edmFunction.GetType().GetProperty("ReturnParameters");
+ return returnParamsProperty == null
+ ? edmFunction.ReturnParameter
+ : ((IEnumerable)returnParamsProperty.GetValue(edmFunction, null)).FirstOrDefault();
+ }
+
+ public bool IsComposable(EdmFunction edmFunction)
+ {
+ ArgumentNotNull(edmFunction, "edmFunction");
+
+ var isComposableProperty = edmFunction.GetType().GetProperty("IsComposableAttribute");
+ return isComposableProperty != null && (bool)isComposableProperty.GetValue(edmFunction, null);
+ }
+
+ public IEnumerable GetParameters(EdmFunction edmFunction)
+ {
+ return FunctionImportParameter.Create(edmFunction.Parameters, _code, _ef);
+ }
+
+ public TypeUsage GetReturnType(EdmFunction edmFunction)
+ {
+ var returnParam = GetReturnParameter(edmFunction);
+ return returnParam == null ? null : _ef.GetElementType(returnParam.TypeUsage);
+ }
+
+ public bool GenerateMergeOptionFunction(EdmFunction edmFunction, bool includeMergeOption)
+ {
+ var returnType = GetReturnType(edmFunction);
+ return !includeMergeOption && returnType != null && returnType.EdmType.BuiltInTypeKind == BuiltInTypeKind.EntityType;
+ }
+}
+
+public static void ArgumentNotNull(T arg, string name) where T : class
+{
+ if (arg == null)
+ {
+ throw new ArgumentNullException(name);
+ }
+}
+#>
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/RedditDatabaseConfiguration.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/RedditDatabaseConfiguration.cs
new file mode 100644
index 000000000..461c5c039
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/RedditDatabaseConfiguration.cs
@@ -0,0 +1,13 @@
+using System.Data.Entity;
+using System.Data.Entity.SqlServer;
+
+namespace RedditCore.DataModel
+{
+ public class RedditDatabaseConfiguration : DbConfiguration
+ {
+ public RedditDatabaseConfiguration()
+ {
+ SetExecutionStrategy("System.Data.SqlClient", () => new SqlAzureExecutionStrategy());
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/CommentRepository.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/CommentRepository.cs
new file mode 100644
index 000000000..a30951eba
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/CommentRepository.cs
@@ -0,0 +1,39 @@
+using System.Collections.Generic;
+using System.Data;
+using System.Data.Entity;
+using System.Data.Entity.Core.EntityClient;
+using System.Linq;
+using Ninject;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class CommentRepository : RepositoryBase
+ {
+ private readonly IDocumentRemover documentRemover;
+
+ [Inject]
+ public CommentRepository(EntityConnection connection, ILog log, IDocumentRemover documentRemover)
+ : base(connection, log, "comment")
+ {
+ this.documentRemover = documentRemover;
+ }
+
+ protected override IEnumerable PreSave(IEnumerable items, RedditEntities db)
+ {
+ // Delete all comments. Even the invalid comments.
+ // A comment may be made on Reddit, ingested by us, then deleted.
+ // On the next poll this comment will be seen as an InvalidComment. We want
+ // to remove it from the DB along with all the other comments.
+ this.documentRemover.RemoveDocuments(items.ToList());
+
+ // Do not write the invalid comments.
+ return items.Where(x => !(x is InvalidComment));
+ }
+
+ protected override DbSet GetDbSet(RedditEntities db)
+ {
+ return db.Comments;
+ }
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/DbConnectionFactory.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/DbConnectionFactory.cs
new file mode 100644
index 000000000..1d27d2489
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/DbConnectionFactory.cs
@@ -0,0 +1,20 @@
+using System.Data;
+using System.Data.SqlClient;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class DbConnectionFactory : IDbConnectionFactory
+ {
+ private readonly IConfiguration configuration;
+
+ public DbConnectionFactory(IConfiguration configuration)
+ {
+ this.configuration = configuration;
+ }
+
+ public IDbConnection CreateDbConnection()
+ {
+ return new SqlConnection(this.configuration.DbConnectionString);
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/DocumentRemover.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/DocumentRemover.cs
new file mode 100644
index 000000000..a9ee8a678
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/DocumentRemover.cs
@@ -0,0 +1,53 @@
+using System.Collections.Generic;
+using System.Data;
+using Ninject;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class DocumentRemover : IDocumentRemover
+ {
+ private readonly IDbConnectionFactory connectionFactory;
+ private readonly ILog log;
+
+ [Inject]
+ public DocumentRemover(IDbConnectionFactory connectionFactory, ILog log)
+ {
+ this.log = log;
+ this.connectionFactory = connectionFactory;
+ }
+
+ public void RemoveDocuments(IList documents)
+ {
+ log.Verbose($"Starting removal of {documents.Count} documents");
+ var ids = new DataTable();
+ ids.Columns.Add("id", typeof(string));
+
+ foreach (var item in documents)
+ {
+ var row = ids.NewRow();
+ row["id"] = item.Id;
+ ids.Rows.Add(row);
+ }
+
+ using (IDbConnection connection = connectionFactory.CreateDbConnection())
+ {
+ connection.Open();
+
+ using (var command = connection.CreateCommand())
+ {
+ command.CommandText = "reddit.DeleteDocuments";
+ command.CommandType = CommandType.StoredProcedure;
+ var param = command.CreateParameter();
+ param.ParameterName = "DocIds";
+ param.Value = ids;
+ command.Parameters.Add(param);
+
+ command.ExecuteNonQuery();
+ }
+ }
+
+ log.Verbose("Finished document removal");
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/EmbeddedUrlRepository.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/EmbeddedUrlRepository.cs
new file mode 100644
index 000000000..4b585264b
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/EmbeddedUrlRepository.cs
@@ -0,0 +1,21 @@
+using System.Data.Entity;
+using System.Data.Entity.Core.EntityClient;
+using Ninject;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class EmbeddedUrlRepository : RepositoryBase
+ {
+ [Inject]
+ public EmbeddedUrlRepository(EntityConnection connection, ILog log)
+ : base(connection, log, "embeddedUrl")
+ {
+ }
+
+ protected override DbSet GetDbSet(RedditEntities db)
+ {
+ return db.EmbeddedUrls;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IDbConnectionFactory.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IDbConnectionFactory.cs
new file mode 100644
index 000000000..595eba751
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IDbConnectionFactory.cs
@@ -0,0 +1,9 @@
+using System.Data;
+
+namespace RedditCore.DataModel.Repositories
+{
+ public interface IDbConnectionFactory
+ {
+ IDbConnection CreateDbConnection();
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IDocumentRemover.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IDocumentRemover.cs
new file mode 100644
index 000000000..fb8cd2c79
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IDocumentRemover.cs
@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal interface IDocumentRemover
+ {
+ void RemoveDocuments(IList documents);
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IRepository.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IRepository.cs
new file mode 100644
index 000000000..d10907807
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/IRepository.cs
@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+
+namespace RedditCore.DataModel.Repositories
+{
+ public interface IRepository where T : class
+ {
+ void Save(T item);
+
+ void Save(IEnumerable items);
+
+ IList GetAll();
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/PostCommentCountRepository.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/PostCommentCountRepository.cs
new file mode 100644
index 000000000..a89b8491a
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/PostCommentCountRepository.cs
@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Data.Entity;
+using System.Data.Entity.Core.EntityClient;
+using System.Linq;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class PostCommentCountRepository : RepositoryBase
+ {
+ public PostCommentCountRepository(EntityConnection connection, ILog log)
+ : base(connection, log, "PostCommentCount")
+ {
+ }
+
+ protected override IEnumerable PreSave(IEnumerable items, RedditEntities db)
+ {
+ var ids = items.Select(x => x.PostId);
+ var existingItems = (from postCommentCount in db.PostCommentCounts
+ where ids.Contains(postCommentCount.PostId)
+ select postCommentCount).ToList();
+ db.PostCommentCounts.RemoveRange(existingItems);
+ db.SaveChanges();
+
+ return items;
+ }
+
+ protected override DbSet GetDbSet(RedditEntities db)
+ {
+ return db.PostCommentCounts;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/PostRepository.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/PostRepository.cs
new file mode 100644
index 000000000..9435df084
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/PostRepository.cs
@@ -0,0 +1,34 @@
+using System.Collections.Generic;
+using System.Data.Entity;
+using System.Data.Entity.Core.EntityClient;
+using System.Linq;
+using Ninject;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class PostRepository : RepositoryBase
+ {
+ private readonly IDocumentRemover documentRemover;
+
+ [Inject]
+ public PostRepository(EntityConnection connection, ILog log, IDocumentRemover documentRemover)
+ : base(connection, log, "post")
+ {
+ this.documentRemover = documentRemover;
+ }
+
+ protected override IEnumerable PreSave(IEnumerable items, RedditEntities db)
+ {
+ documentRemover.RemoveDocuments(items.ToList());
+
+ // No need to filter posts.
+ return items;
+ }
+
+ protected override DbSet GetDbSet(RedditEntities db)
+ {
+ return db.Posts;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/RepositoryBase.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/RepositoryBase.cs
new file mode 100644
index 000000000..0a9f791aa
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/RepositoryBase.cs
@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Entity;
+using System.Data.Entity.Core.EntityClient;
+using System.Data.Entity.Validation;
+using System.Linq;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal abstract class RepositoryBase : IRepository where T : class
+ {
+ private readonly string itemType;
+
+ internal RepositoryBase(EntityConnection connection, ILog log, string itemType)
+ {
+ Connection = connection;
+ Log = log;
+ this.itemType = itemType;
+ }
+
+ protected ILog Log { get; }
+ protected EntityConnection Connection { get; }
+
+ public virtual void Save(T item)
+ {
+ Save(new List {item});
+ }
+
+ ///
+ /// Pre-save hook. Allows user to filter items before the actual write.
+ ///
+ ///
+ ///
+ /// Items that will be written to the database
+ protected virtual IEnumerable PreSave(IEnumerable items, RedditEntities db)
+ {
+ return items;
+ }
+
+ public virtual void Save(IEnumerable items)
+ {
+ if (items.Any())
+ try
+ {
+ using (var db = new RedditEntities(Connection))
+ {
+ var filtered = PreSave(items, db);
+
+ GetDbSet(db).AddRange(filtered);
+
+ db.SaveChanges();
+ }
+ }
+ catch (DbEntityValidationException ex)
+ {
+ var list = string.Join(",", items);
+ Log.Error($"Error validating item of type {itemType}: {list}", ex);
+ var errors =
+ ex.EntityValidationErrors.Select(
+ e => string.Join(Environment.NewLine,
+ e.ValidationErrors.Select(v => $"{v.PropertyName} - {v.ErrorMessage}")) +
+ " Entity: " + e.Entry.Entity);
+
+ foreach (var error in errors)
+ Log.Error($"----Validation error: {error}");
+ }
+ catch (Exception e)
+ {
+ var list = string.Join(",", items);
+ Log.Error($"Error writing items of type {itemType}: {list}", e);
+ }
+ }
+
+ public virtual IList GetAll()
+ {
+ using (var db = new RedditEntities(Connection))
+ {
+ return new List(GetDbSet(db)).AsReadOnly();
+ }
+ }
+
+ protected abstract DbSet GetDbSet(RedditEntities db);
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/UserDefinedEntityDefinitionRepository.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/UserDefinedEntityDefinitionRepository.cs
new file mode 100644
index 000000000..d23861cb5
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/UserDefinedEntityDefinitionRepository.cs
@@ -0,0 +1,21 @@
+using System.Data.Entity;
+using System.Data.Entity.Core.EntityClient;
+using Ninject;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class UserDefinedEntityDefinitionRepository : RepositoryBase
+ {
+ [Inject]
+ public UserDefinedEntityDefinitionRepository(EntityConnection connection, ILog log)
+ : base(connection, log, "userDefinedEntityDefinition")
+ {
+ }
+
+ protected override DbSet GetDbSet(RedditEntities db)
+ {
+ return db.UserDefinedEntityDefinitions;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/UserDefinedEntityRepository.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/UserDefinedEntityRepository.cs
new file mode 100644
index 000000000..61d1bcc9c
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/Repositories/UserDefinedEntityRepository.cs
@@ -0,0 +1,21 @@
+using System.Data.Entity;
+using System.Data.Entity.Core.EntityClient;
+using Ninject;
+using RedditCore.Logging;
+
+namespace RedditCore.DataModel.Repositories
+{
+ internal class UserDefinedEntityRepository : RepositoryBase
+ {
+ [Inject]
+ public UserDefinedEntityRepository(EntityConnection connection, ILog log)
+ : base(connection, log, "userDefinedEntity")
+ {
+ }
+
+ protected override DbSet GetDbSet(RedditEntities db)
+ {
+ return db.UserDefinedEntities;
+ }
+ }
+}
\ No newline at end of file
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/SocialGistPostId.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/SocialGistPostId.cs
new file mode 100644
index 000000000..fc60d921b
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/SocialGistPostId.cs
@@ -0,0 +1,46 @@
+using System;
+
+namespace RedditCore.DataModel
+{
+ ///
+ /// Contains all fields needed to provide to SocialGist to identify a particular post
+ ///
+ public class SocialGistPostId : IComparable
+ {
+ ///
+ /// Gets or sets the URL of the post/comment
+ ///
+ public string Url { get; set; }
+
+ public override string ToString()
+ {
+ return $"Url {Url}";
+ }
+
+ protected bool Equals(SocialGistPostId other)
+ {
+ return string.Equals(Url, other.Url);
+ }
+
+ public override bool Equals(object obj)
+ {
+ if (ReferenceEquals(null, obj)) return false;
+ if (ReferenceEquals(this, obj)) return true;
+ if (obj.GetType() != this.GetType()) return false;
+ return Equals((SocialGistPostId) obj);
+ }
+
+ public override int GetHashCode()
+ {
+ return (Url != null ? Url.GetHashCode() : 0);
+ }
+
+ public int CompareTo(object obj)
+ {
+ if (ReferenceEquals(null, obj)) return 1;
+ if (obj.GetType() != this.GetType()) return 1;
+ var other = (SocialGistPostId) obj;
+ return string.Compare(Url, other.Url, StringComparison.Ordinal);
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntity.Partial.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntity.Partial.cs
new file mode 100644
index 000000000..5ff734827
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntity.Partial.cs
@@ -0,0 +1,65 @@
+using System;
+using System.Reflection;
+using System.Text;
+
+namespace RedditCore.DataModel
+{
+ public partial class UserDefinedEntity : IEquatable
+ {
+ public override string ToString()
+ {
+ var flags = BindingFlags.Instance | BindingFlags.Public |
+ BindingFlags.FlattenHierarchy;
+ var infos = GetType().GetProperties(flags);
+
+ var sb = new StringBuilder();
+
+ var typeName = GetType().Name;
+ sb.Append(typeName);
+
+ sb.Append("[");
+ foreach (var info in infos)
+ {
+ var value = info.GetValue(this, null);
+ sb.AppendFormat("{0}: {1},", info.Name, value != null ? value : "null");
+ }
+ sb.Append("]");
+
+ return sb.ToString();
+ }
+
+ public bool Equals(UserDefinedEntity other)
+ {
+ if (ReferenceEquals(null, other)) return false;
+ if (ReferenceEquals(this, other)) return true;
+ return DocumentId == other.DocumentId
+ && string.Equals(Entity, other.Entity, StringComparison.OrdinalIgnoreCase)
+ && string.Equals(EntityType, other.EntityType, StringComparison.OrdinalIgnoreCase)
+ && EntityOffset == other.EntityOffset
+ && EntityLength == other.EntityLength
+ && Id == other.Id;
+ }
+
+ public override bool Equals(object obj)
+ {
+ if (ReferenceEquals(null, obj)) return false;
+ if (ReferenceEquals(this, obj)) return true;
+ if (obj.GetType() != this.GetType()) return false;
+ return Equals((UserDefinedEntity) obj);
+ }
+
+ public override int GetHashCode()
+ {
+ unchecked
+ {
+ var hashCode = DocumentId.GetHashCode();
+ hashCode = (hashCode * 397) ^ (Entity != null ? Entity.GetHashCode() : 0);
+ hashCode = (hashCode * 397) ^ (EntityType != null ? EntityType.GetHashCode() : 0);
+ hashCode = (hashCode * 397) ^ EntityOffset.GetHashCode();
+ hashCode = (hashCode * 397) ^ EntityLength.GetHashCode();
+ hashCode = (hashCode * 397) ^ Id.GetHashCode();
+ return hashCode;
+ }
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntity.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntity.cs
new file mode 100644
index 000000000..ffc9c4c50
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntity.cs
@@ -0,0 +1,35 @@
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
+namespace RedditCore.DataModel
+{
+
+using System;
+ using System.Collections.Generic;
+
+public partial class UserDefinedEntity
+{
+
+ public string DocumentId { get; set; }
+
+ public string Entity { get; set; }
+
+ public string EntityType { get; set; }
+
+ public Nullable EntityOffset { get; set; }
+
+ public Nullable EntityLength { get; set; }
+
+ public long Id { get; set; }
+
+}
+
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntityDefinition.Partial.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntityDefinition.Partial.cs
new file mode 100644
index 000000000..39a02cc63
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntityDefinition.Partial.cs
@@ -0,0 +1,30 @@
+using System.Reflection;
+using System.Text;
+
+namespace RedditCore.DataModel
+{
+ public partial class UserDefinedEntityDefinition
+ {
+ public override string ToString()
+ {
+ var flags = BindingFlags.Instance | BindingFlags.Public |
+ BindingFlags.FlattenHierarchy;
+ var infos = GetType().GetProperties(flags);
+
+ var sb = new StringBuilder();
+
+ var typeName = GetType().Name;
+ sb.Append(typeName);
+
+ sb.Append("[");
+ foreach (var info in infos)
+ {
+ var value = info.GetValue(this, null);
+ sb.AppendFormat("{0}: {1},", info.Name, value != null ? value : "null");
+ }
+ sb.Append("]");
+
+ return sb.ToString();
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntityDefinition.cs b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntityDefinition.cs
new file mode 100644
index 000000000..2575a6e13
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DataModel/UserDefinedEntityDefinition.cs
@@ -0,0 +1,31 @@
+
+//------------------------------------------------------------------------------
+//
+// This code was generated from a template.
+//
+// Manual changes to this file may cause unexpected behavior in your application.
+// Manual changes to this file will be overwritten if the code is regenerated.
+//
+//------------------------------------------------------------------------------
+
+
+namespace RedditCore.DataModel
+{
+
+using System;
+ using System.Collections.Generic;
+
+public partial class UserDefinedEntityDefinition
+{
+
+ public string Regex { get; set; }
+
+ public string EntityType { get; set; }
+
+ public string EntityValue { get; set; }
+
+ public string Color { get; set; }
+
+}
+
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DateTimes.cs b/Functions/Code/Reddit/Src/RedditCore/DateTimes.cs
new file mode 100644
index 000000000..e737b40cf
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DateTimes.cs
@@ -0,0 +1,15 @@
+using System;
+
+namespace RedditCore
+{
+ public static class DateTimes
+ {
+ public static DateTime UnixTimeStampToDateTime(long unixTimeStamp)
+ {
+ // Unix timestamp is seconds past epoch
+ DateTime dtDateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
+ dtDateTime = dtDateTime.AddSeconds(unixTimeStamp).ToLocalTime();
+ return dtDateTime;
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/CommentCountAggregator.cs b/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/CommentCountAggregator.cs
new file mode 100644
index 000000000..9d762e970
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/CommentCountAggregator.cs
@@ -0,0 +1,53 @@
+using System.Collections.Generic;
+using System.Linq;
+using RedditCore.DataModel;
+using RedditCore.DataModel.Repositories;
+
+namespace RedditCore.DocumentAggregators
+{
+ internal class CommentCountAggregator : IDocumentAggregator
+ {
+ private readonly IRepository repository;
+
+ public CommentCountAggregator(IRepository repository)
+ {
+ this.repository = repository;
+ }
+
+ public void Aggregate(IEnumerable posts, IEnumerable comments)
+ {
+ var threadCommentCount = new Dictionary();
+
+ if (posts != null)
+ {
+ // Set all comment counts to zero incase a post exists without a comment
+ foreach (var post in posts)
+ {
+ threadCommentCount.Add(post.Id, 0);
+ }
+ }
+
+ if (comments != null)
+ {
+ foreach (var comment in comments)
+ {
+ if (!threadCommentCount.TryGetValue(comment.PostId, out var value))
+ {
+ value = 0;
+ }
+
+ threadCommentCount[comment.PostId] = value + 1;
+ }
+ }
+
+ var results = threadCommentCount.Select(post =>
+ new PostCommentCount()
+ {
+ PostId = post.Key,
+ CommentCount = post.Value
+ });
+
+ this.repository.Save(results);
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/DocumentAggregatorsModule.cs b/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/DocumentAggregatorsModule.cs
new file mode 100644
index 000000000..7edfdc97d
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/DocumentAggregatorsModule.cs
@@ -0,0 +1,12 @@
+using Ninject.Modules;
+
+namespace RedditCore.DocumentAggregators
+{
+ internal class DocumentAggregatorsModule : NinjectModule
+ {
+ public override void Load()
+ {
+ Bind().To();
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/IDocumentAggregator.cs b/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/IDocumentAggregator.cs
new file mode 100644
index 000000000..bf91a0218
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DocumentAggregators/IDocumentAggregator.cs
@@ -0,0 +1,10 @@
+using RedditCore.DataModel;
+using System.Collections.Generic;
+
+namespace RedditCore.DocumentAggregators
+{
+ public interface IDocumentAggregator
+ {
+ void Aggregate(IEnumerable posts, IEnumerable comments);
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/ContainsUserDefinedEntitiesFilter.cs b/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/ContainsUserDefinedEntitiesFilter.cs
new file mode 100644
index 000000000..2dfb9566e
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/ContainsUserDefinedEntitiesFilter.cs
@@ -0,0 +1,22 @@
+using System.Linq;
+using RedditCore.DataModel;
+using RedditCore.SocialGist;
+
+namespace RedditCore.DocumentFilters
+{
+ internal class ContainsUserDefinedEntitiesFilter : IDocumentFilter
+ {
+ private readonly IUserDefinedEntityFinder userDefinedEntityFinder;
+
+ public ContainsUserDefinedEntitiesFilter( IUserDefinedEntityFinder userDefinedEntityFinder)
+ {
+ this.userDefinedEntityFinder = userDefinedEntityFinder;
+ }
+
+ public bool ShouldKeep(IDocument document)
+ {
+ // If the document is an invalid comment then we want to keep it around. This allows us to delete invalid comments from the database later.
+ return (document is InvalidComment) || userDefinedEntityFinder.FindAllUserDefinedEntities(new[] { document }).Any();
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/DocumentFiltersModule.cs b/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/DocumentFiltersModule.cs
new file mode 100644
index 000000000..264dc6819
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/DocumentFiltersModule.cs
@@ -0,0 +1,22 @@
+using Ninject.Modules;
+
+namespace RedditCore.DocumentFilters
+{
+ internal class DocumentFiltersModule : NinjectModule
+ {
+ private readonly bool ingestOnlyUserDefinedEntityDocuments;
+
+ internal DocumentFiltersModule(IConfiguration configuration)
+ {
+ this.ingestOnlyUserDefinedEntityDocuments = configuration.IngestOnlyDocumentsWithUserDefinedEntities;
+ }
+
+ public override void Load()
+ {
+ if(this.ingestOnlyUserDefinedEntityDocuments)
+ {
+ Bind().To();
+ }
+ }
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/IDocumentFilter.cs b/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/IDocumentFilter.cs
new file mode 100644
index 000000000..df3f1c4c8
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/DocumentFilters/IDocumentFilter.cs
@@ -0,0 +1,14 @@
+using RedditCore.DataModel;
+
+namespace RedditCore.DocumentFilters
+{
+ public interface IDocumentFilter
+ {
+ ///
+ /// Does the actual filter computation.
+ ///
+ ///
+ /// True if the document should be filtered. False otherwise.
+ bool ShouldKeep(IDocument document);
+ }
+}
diff --git a/Functions/Code/Reddit/Src/RedditCore/Http/HttpClient.cs b/Functions/Code/Reddit/Src/RedditCore/Http/HttpClient.cs
new file mode 100644
index 000000000..7ce9f7157
--- /dev/null
+++ b/Functions/Code/Reddit/Src/RedditCore/Http/HttpClient.cs
@@ -0,0 +1,273 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Ninject;
+using RedditCore.Logging;
+using RedditCore.Properties;
+using System.Collections.Generic;
+using RedditCore.Telemetry;
+using System.Threading;
+using Microsoft.ApplicationInsights.DataContracts;
+
+namespace RedditCore.Http
+{
+ internal class HttpClient : IHttpClient
+ {
+ private readonly ILog log;
+ private readonly IObjectLogger objectLogger;
+ private readonly ITelemetryClient telemetryClient;
+
+ private const int RETRY_COUNT = 5;
+
+ private readonly TimeSpan minWaitBeforeRetry = TimeSpan.FromSeconds(15);
+ private readonly TimeSpan maxWaitBeforeRetry = TimeSpan.FromSeconds(45);
+
+ [Inject]
+ public HttpClient(
+ ILog log,
+ ITelemetryClient telemetryClient,
+ IObjectLogger objectLogger)
+ {
+ this.log = log;
+ this.objectLogger = objectLogger;
+ this.telemetryClient = telemetryClient;
+ }
+
+ public async Task> GetJsonAsync(
+ Uri requestUri,
+ TimeSpan? requestTimeout = null,
+ AuthenticationHeaderValue authenticationHeaderValue = null)
+ {
+ // SocialGist only accepts TLS 1.1 or higher. AzureFunctions (on Azure) default to SSL3 or TLS
+ // AzureFunctions runing locally use defaults that work with SocialGist but remote Azure Functions do not.
+ ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;
+ log.Verbose($"Retrieving URL using protocol: {ServicePointManager.SecurityProtocol}");
+ telemetryClient.TrackEvent(TelemetryNames.HTTP_Get, new Dictionary { { "URL", requestUri.ToString() } });
+
+ var handler = new HttpClientHandler()
+ {
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
+ };
+
+ using (var client = new System.Net.Http.HttpClient(handler))
+ {
+ if (requestTimeout.HasValue)
+ {
+ client.Timeout = requestTimeout.Value;
+ }
+
+ if (authenticationHeaderValue != null)
+ {
+ client.DefaultRequestHeaders.Authorization = authenticationHeaderValue;
+ }
+
+ string data = "";
+ HttpResponseMessage response = null;
+ int retryCount = 0;
+ while (retryCount < RETRY_COUNT)
+ {
+ try
+ {
+ telemetryClient.TrackEvent(TelemetryNames.HTTP_Try, new Dictionary { { "URL", requestUri?.ToString() }, { "try number", retryCount.ToString() } });
+ log.Verbose($"Retrieving URL {requestUri} on attempt {retryCount}");
+
+ // Track telemetry for each host query
+ using (var tracker = telemetryClient.StartTrackDependency(requestUri.Host, requestUri.AbsolutePath, "HTTP"))
+ {
+ tracker.Properties.Add("Request URL", requestUri?.ToString());
+ tracker.Properties.Add("Try attempt", retryCount.ToString());
+
+ try
+ {
+ response = await client.GetAsync(requestUri);
+ }
+ catch(TaskCanceledException)
+ {
+ tracker.IsSuccess = false;
+ tracker.ResultCode = "Task Canceled (HTTP Timeout)";
+ throw;
+ }
+ catch(HttpRequestException e)
+ {
+ tracker.IsSuccess = false;
+ tracker.ResultCode = $"HTTPException: {e.Message}";
+ throw;
+ }
+
+ tracker.IsSuccess = response.IsSuccessStatusCode;
+ tracker.ResultCode = response.StatusCode.ToString();
+
+ // Get data after setting telemetry status codes incase this errors out.
+ data = await response.Content.ReadAsStringAsync();
+
+ if(!response.IsSuccessStatusCode)
+ {
+ // Do not store this data unless the request failed. This data can get very large very fast.
+ tracker.Properties.Add("Server Response", data);
+ }
+ }
+
+ if (response.IsSuccessStatusCode)
+ {
+ this.objectLogger.Log(data, "HttpClient", requestUri.ToString());
+
+ // Successfully completed request
+ break;
+ }
+ else
+ {
+ telemetryClient.TrackEvent(TelemetryNames.HTTP_Error, new Dictionary {
+ { "URL", requestUri?.ToString() },
+ { "StatusCode", response?.StatusCode.ToString() }
+ });
+ retryCount = HandleFailure(retryCount, requestUri, response);
+ }
+ }
+ catch (Exception e)
+ {
+ retryCount++;
+ telemetryClient.TrackEvent(TelemetryNames.HTTP_Error, new Dictionary {
+ { "URL", requestUri?.ToString() },
+ { "StatusCode", response?.StatusCode.ToString() },
+ { "Exception", e.ToString() }
+ });
+ if (retryCount < RETRY_COUNT)
+ {
+ log.Error($"Error executing web request for URL {requestUri} on attempt {retryCount}. Retrying.", e);
+ }
+ else
+ {
+ log.Error($"Error executing web request for URI {requestUri} on attempt {retryCount}. NOT retrying", e);
+
+ // Rethrow the exception to fail the current job
+ throw;
+ }
+ }
+
+ // Do not wait after the final try.
+ if (retryCount < RETRY_COUNT)
+ {
+ // Request failed. Wait a random time and try again. Random() uses a time-dependent seed.
+ int wait = new Random().Next((int)minWaitBeforeRetry.TotalMilliseconds, (int)maxWaitBeforeRetry.TotalMilliseconds);
+ log.Verbose($"Waiting {wait} milliseconds before retrying");
+ Thread.Sleep(wait);
+ }
+ }
+
+ log.Verbose($"URL {requestUri} retrieved.");
+
+ var metric = new MetricTelemetry();
+ metric.Name = TelemetryNames.HTTP_RetryCount;
+ metric.Sum = retryCount;
+ metric.Timestamp = DateTime.Now;
+ metric.Properties.Add("Domain", requestUri.Host);
+ telemetryClient.TrackMetric(metric);
+
+ try
+ {
+ var obj = JsonConvert.DeserializeObject(data);
+
+ log.Verbose($"Result deserialized from URL {requestUri}");
+
+ return new HttpJsonResponseMessage
+ {
+ Object = obj,
+ ResponseMessage = response
+ };
+ }
+ catch (Exception e)
+ {
+ telemetryClient.TrackEvent(TelemetryNames.HTTP_JSON_Error, new Dictionary
+ {
+ {"URL", requestUri?.ToString()},
+ {"StatusCode", response?.StatusCode.ToString()},
+ {"Exception", e.ToString()},
+ {"Response", data}
+ });
+
+ // Log and rethrow the exception
+ log.Error($"Error deserializing result from URL {requestUri}", e);
+
+ // Rethrow the exception to fail the current job
+ throw;
+ }
+ }
+ }
+
+ private int HandleFailure(int retryCount, Uri requestUri, HttpResponseMessage response)
+ {
+ var result = retryCount + 1;
+ if (result < RETRY_COUNT)
+ {
+ log.Warning($"Error executing web request for URL {requestUri} on attempt {result}. Response code: {response.StatusCode}. Retrying.");
+ }
+ else
+ {
+ log.Warning($"Error executing web request for URI {requestUri} on attempt {result}. Response code: {response.StatusCode}. NOT retrying");
+ }
+
+ return result;
+ }
+
+ public async Task