Skip to content

Commit

Permalink
Fixes #127: Added ability to record http headers (#300)
Browse files Browse the repository at this point in the history 
* Fixes #127: Added ability to record http headers

* Normalized request info collectors

* Added TODO for getting web api post data
  • Loading branch information
@niemyjski
niemyjski authored Apr 25, 2023
1 parent 9e24a9c commit 670ead2
Show file tree
Hide file tree
Showing 5 changed files with 189 additions and 56 deletions.
6 changes: 6 additions & 0 deletions src/Exceptionless/Configuration/ExceptionlessConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ public bool IncludePrivateInformation {
IncludeUserName = value;
IncludeMachineName = value;
IncludeIpAddress = value;
IncludeHeaders = value;
IncludeCookies = value;
IncludePostData = value;
IncludeQueryString = value;
Expand All @@ -247,6 +248,11 @@ public bool IncludePrivateInformation {
/// </summary>
public bool IncludeIpAddress { get; set; }
/// <summary>
/// Gets or sets a value indicating whether to include Headers.
/// NOTE: DataExclusions are applied to all Headers keys when enabled.
/// </summary>
public bool IncludeHeaders { get; set; }
/// <summary>
/// Gets or sets a value indicating whether to include Cookies.
/// NOTE: DataExclusions are applied to all Cookie keys when enabled.
/// </summary>
Expand Down
9 changes: 8 additions & 1 deletion src/Exceptionless/Models/Client/Data/RequestInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ namespace Exceptionless.Models.Data {
public class RequestInfo : IData {
public RequestInfo() {
Data = new DataDictionary();
Headers = new Dictionary<string, string[]>();
Cookies = new Dictionary<string, string>();
QueryString = new Dictionary<string, string>();
}
Expand Down Expand Up @@ -49,6 +50,11 @@ public RequestInfo() {
/// </summary>
public string ClientIpAddress { get; set; }

/// <summary>
/// The header values from the request.
/// </summary>
public Dictionary<string, string[]> Headers { get; set; }

/// <summary>
/// The request cookies.
/// </summary>
Expand All @@ -70,7 +76,7 @@ public RequestInfo() {
public DataDictionary Data { get; set; }

protected bool Equals(RequestInfo other) {
return string.Equals(UserAgent, other.UserAgent) && string.Equals(HttpMethod, other.HttpMethod) && IsSecure == other.IsSecure && string.Equals(Host, other.Host) && Port == other.Port && string.Equals(Path, other.Path) && string.Equals(Referrer, other.Referrer) && string.Equals(ClientIpAddress, other.ClientIpAddress) && Cookies.CollectionEquals(other.Cookies) && QueryString.CollectionEquals(other.QueryString) && Equals(Data, other.Data);
return string.Equals(UserAgent, other.UserAgent) && string.Equals(HttpMethod, other.HttpMethod) && IsSecure == other.IsSecure && string.Equals(Host, other.Host) && Port == other.Port && string.Equals(Path, other.Path) && string.Equals(Referrer, other.Referrer) && string.Equals(ClientIpAddress, other.ClientIpAddress) && Headers.CollectionEquals(other.Headers) && Cookies.CollectionEquals(other.Cookies) && QueryString.CollectionEquals(other.QueryString) && Equals(Data, other.Data);
}

public override bool Equals(object obj) {
Expand All @@ -95,6 +101,7 @@ public override int GetHashCode() {
hashCode = (hashCode * 397) ^ (Path == null ? 0 : Path.GetHashCode());
hashCode = (hashCode * 397) ^ (Referrer == null ? 0 : Referrer.GetHashCode());
hashCode = (hashCode * 397) ^ (ClientIpAddress == null ? 0 : ClientIpAddress.GetHashCode());
hashCode = (hashCode * 397) ^ (Headers == null ? 0 : Headers.GetCollectionHashCode());
hashCode = (hashCode * 397) ^ (Cookies == null ? 0 : Cookies.GetCollectionHashCode(_cookieHashCodeExclusions));
hashCode = (hashCode * 397) ^ (QueryString == null ? 0 : QueryString.GetCollectionHashCode());
hashCode = (hashCode * 397) ^ (Data == null ? 0 : Data.GetCollectionHashCode());
Expand Down
70 changes: 53 additions & 17 deletions src/Platforms/Exceptionless.AspNetCore/RequestInfoCollector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,16 @@
namespace Exceptionless.AspNetCore {
public static class RequestInfoCollector {
private const int MAX_BODY_SIZE = 50 * 1024;
private const int MAX_DATA_ITEM_LENGTH = 1000;

public static RequestInfo Collect(HttpContext context, ExceptionlessConfiguration config) {
if (context == null)
return null;

var info = new RequestInfo {
HttpMethod = context.Request.Method,
IsSecure = context.Request.IsHttps,
Path = context.Request.Path.HasValue ? context.Request.Path.Value : "/",
Path = context.Request.Path.HasValue ? context.Request.Path.Value : "/"
};

if (config.IncludeIpAddress)
Expand All @@ -32,13 +34,16 @@ public static RequestInfo Collect(HttpContext context, ExceptionlessConfiguratio

info.Port = context.Request.Host.Port.GetValueOrDefault(info.IsSecure ? 443 : 80);

if (context.Request.Headers.ContainsKey(HeaderNames.UserAgent))
info.UserAgent = context.Request.Headers[HeaderNames.UserAgent].ToString();
if (context.Request.Headers.TryGetValue(HeaderNames.UserAgent, out var userAgentHeader))
info.UserAgent = userAgentHeader.ToString();

if (context.Request.Headers.ContainsKey(HeaderNames.Referer))
info.Referrer = context.Request.Headers[HeaderNames.Referer].ToString();
if (context.Request.Headers.TryGetValue(HeaderNames.Referer, out var refererHeader))
info.Referrer = refererHeader.ToString();

var exclusionList = config.DataExclusions as string[] ?? config.DataExclusions.ToArray();
if (config.IncludeHeaders)
info.Headers = context.Request.Headers.ToHeaderDictionary(exclusionList);

if (config.IncludeCookies)
info.Cookies = context.Request.Cookies.ToDictionary(exclusionList);

Expand Down Expand Up @@ -93,17 +98,15 @@ private static object GetPostData(HttpContext context, ExceptionlessConfiguratio
return message;
}

var maxDataToRead = contentLength == 0 ? MAX_BODY_SIZE : contentLength;

// pass default values, except for leaveOpen: true. This prevents us from disposing the underlying stream
using (var inputStream = new StreamReader(context.Request.Body, Encoding.UTF8, true, 1024, true)) {
var sb = new StringBuilder();
int numRead;

int bufferSize = (int)Math.Min(1024, maxDataToRead);
int bufferSize = (int)Math.Min(1024, contentLength);

char[] buffer = new char[bufferSize];
while ((numRead = inputStream.ReadBlock(buffer, 0, bufferSize)) > 0 && (sb.Length + numRead) <= maxDataToRead) {
while ((numRead = inputStream.ReadBlock(buffer, 0, bufferSize)) > 0 && (sb.Length + numRead) <= contentLength) {
sb.Append(buffer, 0, numRead);
}
string postData = sb.ToString();
Expand All @@ -121,8 +124,15 @@ private static object GetPostData(HttpContext context, ExceptionlessConfiguratio
}
}

private static readonly List<string> _ignoredFormFields = new List<string> {
"__*"
private static readonly List<string> _ignoredHeaders = new List<string> {
HeaderNames.Authorization,
HeaderNames.Cookie,
HeaderNames.Host,
HeaderNames.Method,
HeaderNames.Path,
HeaderNames.ProxyAuthorization,
HeaderNames.Referer,
HeaderNames.UserAgent
};

private static readonly List<string> _ignoredCookies = new List<string> {
Expand All @@ -131,20 +141,44 @@ private static object GetPostData(HttpContext context, ExceptionlessConfiguratio
"*SessionId*"
};

private static Dictionary<string, string> ToDictionary(this IRequestCookieCollection cookies, IList<string> exclusions) {
private static readonly List<string> _ignoredFormFields = new List<string> {
"__*"
};

private static Dictionary<string, string[]> ToHeaderDictionary(this IEnumerable<KeyValuePair<string, StringValues>> headers, string[] exclusions) {
var d = new Dictionary<string, string[]>();

foreach (var header in headers) {
if (String.IsNullOrEmpty(header.Key) || _ignoredHeaders.Contains(header.Key) || header.Key.AnyWildcardMatches(exclusions))
continue;

string[] values = header.Value.Where(hv => hv != null && hv.Length < MAX_DATA_ITEM_LENGTH).ToArray();
if (values.Length == 0)
continue;

d[header.Key] = values;
}

return d;
}

private static Dictionary<string, string> ToDictionary(this IRequestCookieCollection cookies, string[] exclusions) {
var d = new Dictionary<string, string>();

foreach (var kvp in cookies) {
if (String.IsNullOrEmpty(kvp.Key) || kvp.Key.AnyWildcardMatches(_ignoredCookies) || kvp.Key.AnyWildcardMatches(exclusions))
continue;

d.Add(kvp.Key, kvp.Value);
if (kvp.Value == null || kvp.Value.Length >= MAX_DATA_ITEM_LENGTH)
continue;

d[kvp.Key] = kvp.Value;
}

return d;
}

private static Dictionary<string, string> ToDictionary(this IEnumerable<KeyValuePair<string, StringValues>> values, IEnumerable<string> exclusions) {
private static Dictionary<string, string> ToDictionary(this IEnumerable<KeyValuePair<string, StringValues>> values, string[] exclusions) {
var d = new Dictionary<string, string>();

foreach (var kvp in values) {
Expand All @@ -153,10 +187,12 @@ private static Dictionary<string, string> ToDictionary(this IEnumerable<KeyValue

try {
string value = kvp.Value.ToString();
d.Add(kvp.Key, value);
if (value.Length >= MAX_DATA_ITEM_LENGTH)
continue;

d[kvp.Key] = value;
} catch (Exception ex) {
if (!d.ContainsKey(kvp.Key))
d.Add(kvp.Key, ex.Message);
d[kvp.Key] = $"EXCEPTION: {ex.Message}";
}
}

Expand Down
84 changes: 61 additions & 23 deletions src/Platforms/Exceptionless.Web/RequestInfoCollector.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
Expand All @@ -13,8 +12,8 @@

namespace Exceptionless.ExtendedData {
internal static class RequestInfoCollector {
private const int MAX_BODY_SIZE = 50 * 1024;
private const int MAX_DATA_ITEM_LENGTH = 1000;
private const int MAX_BODY_SIZE = 50*1024;

public static RequestInfo Collect(HttpContextBase context, ExceptionlessConfiguration config) {
if (context == null)
Expand Down Expand Up @@ -50,13 +49,12 @@ public static RequestInfo Collect(HttpContextBase context, ExceptionlessConfigur
info.Port = context.Request.Url.Port;

var exclusionList = config.DataExclusions as string[] ?? config.DataExclusions.ToArray();
if (config.IncludeHeaders)
info.Headers = context.Request.Headers.ToHeaderDictionary(exclusionList);

if (config.IncludeCookies)
info.Cookies = context.Request.Cookies.ToDictionary(exclusionList);

if (config.IncludePostData && !String.Equals(context.Request.HttpMethod, "GET", StringComparison.OrdinalIgnoreCase))
info.PostData = GetPostData(context, config, exclusionList);


if (config.IncludeQueryString) {
try {
info.QueryString = context.Request.QueryString.ToDictionary(exclusionList);
Expand All @@ -65,6 +63,9 @@ public static RequestInfo Collect(HttpContextBase context, ExceptionlessConfigur
}
}

if (config.IncludePostData && !String.Equals(context.Request.HttpMethod, "GET", StringComparison.OrdinalIgnoreCase))
info.PostData = GetPostData(context, config, exclusionList);

return info;
}

Expand Down Expand Up @@ -139,8 +140,15 @@ private static object GetPostData(HttpContextBase context, ExceptionlessConfigur
}
}

private static readonly List<string> _ignoredFormFields = new List<string> {
"__*"
private static readonly List<string> _ignoredHeaders = new List<string> {
"Authorization",
"Cookie",
"Host",
"Method",
"Path",
"Proxy-Authorization",
"Referer",
"User-Agent"
};

private static readonly List<string> _ignoredCookies = new List<string> {
Expand All @@ -149,38 +157,68 @@ private static object GetPostData(HttpContextBase context, ExceptionlessConfigur
"*SessionId*"
};

private static Dictionary<string, string> ToDictionary(this HttpCookieCollection cookies, IEnumerable<string> exclusions) {
private static readonly List<string> _ignoredFormFields = new List<string> {
"__*"
};

private static Dictionary<string, string[]> ToHeaderDictionary(this NameValueCollection headers, string[] exclusions) {
var result = new Dictionary<string, string[]>();

foreach (string key in headers.AllKeys) {
if (String.IsNullOrEmpty(key) || _ignoredHeaders.Contains(key) || key.AnyWildcardMatches(exclusions))
continue;

try {
string value = headers.Get(key);
if (value == null || value.Length >= MAX_DATA_ITEM_LENGTH)
continue;

result[key] = value.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
}
catch (Exception ex) {
result[key] = new[] { $"EXCEPTION: {ex.Message}" };
}
}

return result;
}

private static Dictionary<string, string> ToDictionary(this HttpCookieCollection cookies, string[] exclusions) {
var d = new Dictionary<string, string>();

foreach (string key in cookies.AllKeys.Distinct().Where(k => !String.IsNullOrEmpty(k) && !k.AnyWildcardMatches(_ignoredCookies) && !k.AnyWildcardMatches(exclusions))) {
foreach (string key in cookies.AllKeys.Distinct()) {
if (String.IsNullOrEmpty(key) || key.AnyWildcardMatches(_ignoredCookies) || key.AnyWildcardMatches(exclusions))
continue;

try {
HttpCookie cookie = cookies.Get(key);
if (cookie != null && cookie.Value != null && cookie.Value.Length < MAX_DATA_ITEM_LENGTH && !d.ContainsKey(key))
d.Add(key, cookie.Value);
var cookie = cookies.Get(key);
if (cookie == null || cookie.Value == null || cookie.Value.Length >= MAX_DATA_ITEM_LENGTH)
continue;

d[key] = cookie.Value;
} catch (Exception ex) {
if (!d.ContainsKey(key))
d.Add(key, ex.Message);
d[key] = $"EXCEPTION: {ex.Message}";
}
}

return d;
}

private static Dictionary<string, string> ToDictionary(this NameValueCollection values, IEnumerable<string> exclusions) {
private static Dictionary<string, string> ToDictionary(this NameValueCollection values, string[] exclusions) {
var d = new Dictionary<string, string>();

var exclusionsArray = exclusions as string[] ?? exclusions.ToArray();

foreach (string key in values.AllKeys) {
if (String.IsNullOrEmpty(key) || key.AnyWildcardMatches(_ignoredFormFields) || key.AnyWildcardMatches(exclusionsArray))
if (String.IsNullOrEmpty(key) || key.AnyWildcardMatches(_ignoredFormFields) || key.AnyWildcardMatches(exclusions))
continue;

try {
string value = values.Get(key);
if (value != null && !d.ContainsKey(key) && value.Length < MAX_DATA_ITEM_LENGTH)
d.Add(key, value);
if (value == null || d.ContainsKey(key) || value.Length >= MAX_DATA_ITEM_LENGTH)
continue;

d[key] = value;
} catch (Exception ex) {
if (!d.ContainsKey(key))
d.Add(key, "EXCEPTION: " + ex.Message);
d[key] = $"EXCEPTION: {ex.Message}";
}
}

Expand Down
Loading

0 comments on commit 670ead2

Please sign in to comment.