From bae16edc23aba943afb5c510056d6cef1bc20b97 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 15 Apr 2021 16:43:43 -0500 Subject: [PATCH] Stack filtering improvements and bug fixes (#825) * #746 - Add more test coverage for resolving new stack counts * Fixed some bugs with test data builder * Update deps. Some k8s updates. * Progress on new stack filter issue. * More progress * Don't apply retention filter to stack id filter when inverted * Got the main test passing. Not sure if the other failures are correct or not. * Fixed one failing unit test * Some minor changes * Disable AD Windows build warnings #493 * Added ability to generate many events using TotalOccurrences * Refactored how additional events are created. * Added new test for posting null session identity name * WIP - Event Stack Filter Tests * Update Deps * Fixed some build messages * Fix issue with message bus broker async fire and forget. * Working on stack inverting issues * Update ES docker to 7.12 * Progress in stack filter refactor * Fix a couple tests * Fixing more tests * Fix remaining tests. Update repos. * Remove repos and parser projects * Update deps / respond to feedback Co-authored-by: Blake Niemyjski --- .github/workflows/build.yaml | 2 +- .gitignore | 2 + Dockerfile | 2 +- Exceptionless.sln | 2 +- build/common.props | 6 +- build/docker/elasticsearch/Dockerfile | 2 +- docker-compose.yml | 4 +- global.json | 2 +- k8s/certificates.yaml | 2 +- k8s/cluster-issuer.yaml | 4 +- k8s/ex-prod-tasks.ps1 | 8 +- k8s/ex-setup.ps1 | 15 +- k8s/vpa-values.yaml | 5 + .../ActiveDirectoryLoginProvider.cs | 6 +- .../Exceptionless.Core.csproj | 20 +- .../Extensions/QueryNodeExtensions.cs | 23 ++ src/Exceptionless.Core/Mail/Mailer.cs | 13 +- src/Exceptionless.Core/Models/Stack.cs | 2 +- .../Models/StackSummaryModel.cs | 2 + .../Configuration/Indexes/EventIndex.cs | 1 - .../Repositories/Queries/AppFilterQuery.cs | 10 +- .../Queries/EventStackFilterQuery.cs | 60 ++- .../Visitors/EventStackFilterQueryVisitor.cs | 374 +++++++----------- .../Visitors/StackDateFixedQueryVisitor.cs | 8 +- .../Validation/IsObjectIdValidator.cs | 4 +- .../Exceptionless.Insulation.csproj | 32 +- .../Exceptionless.Job.csproj | 11 +- .../Controllers/EventController.cs | 123 ++++-- .../Exceptionless.Web.csproj | 16 +- .../Hubs/MessageBusBrokerMiddleware.cs | 4 +- src/Exceptionless.Web/Startup.cs | 6 +- .../Controllers/EventControllerTests.cs | 162 +++++++- .../Exceptionless.Tests.csproj | 10 +- .../Extensions/TestServerExtensions.cs | 12 +- .../IntegrationTestsBase.cs | 6 +- tests/Exceptionless.Tests/Mail/MailerTests.cs | 6 +- .../Search/EventStackFilterQueryTests.cs | 113 ++++++ .../EventStackFilterQueryVisitorTests.cs | 169 ++++++-- .../Search/MoreEventIndexTests.cs | 2 +- .../Utility/DataBuilder.cs | 116 ++++-- 40 files changed, 901 insertions(+), 466 deletions(-) create mode 100644 k8s/vpa-values.yaml create mode 100644 src/Exceptionless.Core/Extensions/QueryNodeExtensions.cs create mode 100644 tests/Exceptionless.Tests/Search/EventStackFilterQueryTests.cs diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ad6c7f6da4..8415ab5c15 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -19,7 +19,7 @@ jobs: run: "echo ref: ${{github.ref}} event: ${{github.event_name}}" - name: Build Version run: | - dotnet tool install --global minver-cli --version 2.3.1 + dotnet tool install --global minver-cli --version 2.5.0 version=$(minver --tag-prefix v) echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV echo "VERSION=$version" >> $GITHUB_ENV diff --git a/.gitignore b/.gitignore index 3ac1ad5e90..78c37efc30 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,5 @@ k8s/ex-prod\.yaml *secrets* k8s/ex-*-snapshots.yaml node_modules + +*.DotSettings diff --git a/Dockerfile b/Dockerfile index 7468722f1e..9dd9371e45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,7 +78,7 @@ CMD [ "dotnet", "Exceptionless.Web.dll" ] # completely self-contained -FROM exceptionless/elasticsearch:7.10.0 AS exceptionless +FROM exceptionless/elasticsearch:7.12.0 AS exceptionless WORKDIR /app COPY --from=api-publish /app/src/Exceptionless.Web/out ./ diff --git a/Exceptionless.sln b/Exceptionless.sln index 4d7d5c5bf2..7c6fc456b6 100644 --- a/Exceptionless.sln +++ b/Exceptionless.sln @@ -13,6 +13,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution CONTRIBUTING.md = CONTRIBUTING.md docker-compose.yml = docker-compose.yml Dockerfile = Dockerfile + build\docker\elasticsearch\Dockerfile = build\docker\elasticsearch\Dockerfile global.json = global.json LICENSE.txt = LICENSE.txt NuGet.Config = NuGet.Config @@ -58,7 +59,6 @@ Global HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1A90AFA5-B81C-4B1B-9DFA-2D90F8CA0EF0} EndGlobalSection EndGlobal diff --git a/build/common.props b/build/common.props index 3839931b50..af4893ae25 100644 --- a/build/common.props +++ b/build/common.props @@ -5,7 +5,7 @@ Exceptionless true v - Copyright (c) 2020 Exceptionless. All rights reserved. + Copyright (c) 2021 Exceptionless. All rights reserved. https://github.com/exceptionless/exceptionless Exceptionless $(NoWarn);CS1591 @@ -19,8 +19,8 @@ - - + + diff --git a/build/docker/elasticsearch/Dockerfile b/build/docker/elasticsearch/Dockerfile index 0481230933..ecb8199ba2 100644 --- a/build/docker/elasticsearch/Dockerfile +++ b/build/docker/elasticsearch/Dockerfile @@ -1,5 +1,5 @@ # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:7.11.0 +FROM docker.elastic.co/elasticsearch/elasticsearch:7.12.0 RUN elasticsearch-plugin install -b mapper-size RUN elasticsearch-plugin install -b repository-azure diff --git a/docker-compose.yml b/docker-compose.yml index e2cb521436..890ee2c66e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ version: '3.5' services: elasticsearch: - image: exceptionless/elasticsearch:7.10.0 + image: exceptionless/elasticsearch:7.12.0 environment: discovery.type: single-node xpack.security.enabled: 'false' @@ -17,7 +17,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:7.10.0 + image: docker.elastic.co/kibana/kibana:7.12.0 ports: - 5601:5601 diff --git a/global.json b/global.json index 22ddf7c56a..208bc74fb9 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { "version": "5.0.100", - "rollForward": "latestMajor" + "rollForward": "latestMinor" } } diff --git a/k8s/certificates.yaml b/k8s/certificates.yaml index 8760d6b2b4..c7d2ae6d81 100644 --- a/k8s/certificates.yaml +++ b/k8s/certificates.yaml @@ -1,4 +1,4 @@ -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: tls-secret diff --git a/k8s/cluster-issuer.yaml b/k8s/cluster-issuer.yaml index eb9b49fc11..f42004e32c 100644 --- a/k8s/cluster-issuer.yaml +++ b/k8s/cluster-issuer.yaml @@ -1,4 +1,4 @@ -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: letsencrypt-prod @@ -14,7 +14,7 @@ spec: class: nginx --- -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: selfsigned diff --git a/k8s/ex-prod-tasks.ps1 b/k8s/ex-prod-tasks.ps1 index d2efc9fb06..58d60363b6 100644 --- a/k8s/ex-prod-tasks.ps1 +++ b/k8s/ex-prod-tasks.ps1 @@ -44,17 +44,16 @@ kubectl run --namespace ex-prod ex-prod-client --rm --tty -i --restart='Never' ` # upgrade nginx ingress to latest # https://github.com/kubernetes/ingress-nginx/releases helm repo update -helm upgrade --reset-values --namespace nginx-ingress -f nginx-values.yaml nginx-ingress stable/nginx-ingress --dry-run +helm upgrade --reset-values --namespace ingress-nginx -f nginx-values.yaml ingress-nginx ingress-nginx/ingress-nginx --dry-run # upgrade cert-manager # https://github.com/jetstack/cert-manager/releases helm repo update -kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.crds.yaml helm upgrade cert-manager jetstack/cert-manager --namespace cert-manager --reset-values --set ingressShim.defaultIssuerName=letsencrypt-prod --set ingressShim.defaultIssuerKind=ClusterIssuer --dry-run # upgrade dashboard # https://github.com/kubernetes/dashboard/releases -kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.3/aio/deploy/recommended.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.1.0/aio/deploy/recommended.yaml # upgrade kubecost helm repo update @@ -63,11 +62,12 @@ helm upgrade kubecost kubecost/cost-analyzer --namespace kubecost --reset-values # upgrade goldilocks helm repo update helm upgrade goldilocks fairwinds-stable/goldilocks --namespace goldilocks --reset-values --dry-run +helm upgrade vpa fairwinds-stable/vpa --namespace vpa -f vpa-values.yaml --reset-values --dry-run # upgrade elasticsearch operator # https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html # https://github.com/elastic/cloud-on-k8s/releases -kubectl apply -f https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/1.3.1/all-in-one.yaml # upgrade elasticsearch kubectl apply --namespace ex-prod -f ex-prod-elasticsearch.yaml diff --git a/k8s/ex-setup.ps1 b/k8s/ex-setup.ps1 index bab04cad66..910e827be8 100644 --- a/k8s/ex-setup.ps1 +++ b/k8s/ex-setup.ps1 @@ -10,7 +10,7 @@ choco install azure-cli # install helm choco install kubernetes-helm -helm repo add stable https://kubernetes-charts.storage.googleapis.com/ +helm repo add "stable" "https://charts.helm.sh/stable" --force-update helm repo add jetstack https://charts.jetstack.io helm repo update @@ -48,7 +48,7 @@ az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER --overwr # install dashboard # https://github.com/kubernetes/dashboard/releases -kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.3/aio/deploy/recommended.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.1.0/aio/deploy/recommended.yaml # create admin user to login to the dashboard kubectl apply -f admin-service-account.yaml @@ -59,7 +59,7 @@ kubectl config set-context --current --namespace=ex-$ENV # setup elasticsearch operator # https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html # https://github.com/elastic/cloud-on-k8s/releases -kubectl apply -f https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/1.3.1/all-in-one.yaml # view ES operator logs kubectl -n elastic-system logs -f statefulset.apps/elastic-operator @@ -98,11 +98,11 @@ curl -X PUT -H "Content-Type: application/json" -k ` Remove-Job $ELASTIC_JOB # install nginx ingress -helm install nginx-ingress stable/nginx-ingress --namespace nginx-ingress --values nginx-values.yaml +helm install --namespace ingress-nginx -f nginx-values.yaml ingress-nginx ingress-nginx/ingress-nginx # wait for external ip to be assigned -kubectl get service -l app=nginx-ingress --namespace nginx-ingress -$IP=$(kubectl get service -l app=nginx-ingress --namespace nginx-ingress -o=jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') +kubectl get service -l app.kubernetes.io/name=ingress-nginx --namespace ingress-nginx +$IP=$(kubectl get service -l app.kubernetes.io/name=ingress-nginx --namespace ingress-nginx -o=jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') $PUBLICIPID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$IP')].[id]" --output tsv) az network public-ip update --ids $PUBLICIPID --dns-name $CLUSTER @@ -117,11 +117,14 @@ helm install cert-manager jetstack/cert-manager --namespace cert-manager --set i # https://kubecost.com/install?ref=home kubectl create namespace kubecost helm repo add kubecost https://kubecost.github.io/cost-analyzer/ +$KUBECOST_KEY="" helm install kubecost kubecost/cost-analyzer --namespace kubecost --set kubecostToken=$KUBECOST_KEY # install goldilocks helm repo add fairwinds-stable https://charts.fairwinds.com/stable +helm install vpa fairwinds-stable/vpa --namespace vpa --create-namespace -f vpa-values.yaml helm install goldilocks fairwinds-stable/goldilocks --namespace goldilocks +kubectl label ns ex-$ENV goldilocks.fairwinds.com/enabled=true # TODO: update this file using the cluster name for the dns kubectl apply -f certificates.yaml diff --git a/k8s/vpa-values.yaml b/k8s/vpa-values.yaml new file mode 100644 index 0000000000..9b58508a1b --- /dev/null +++ b/k8s/vpa-values.yaml @@ -0,0 +1,5 @@ +recommender: + extraArgs: + prometheus-address: | + http://kubecost-prometheus-server.kubecost.svc.cluster.local + storage: prometheus \ No newline at end of file diff --git a/src/Exceptionless.Core/Authentication/ActiveDirectoryLoginProvider.cs b/src/Exceptionless.Core/Authentication/ActiveDirectoryLoginProvider.cs index 0591b3bc0d..e5a92a29f7 100644 --- a/src/Exceptionless.Core/Authentication/ActiveDirectoryLoginProvider.cs +++ b/src/Exceptionless.Core/Authentication/ActiveDirectoryLoginProvider.cs @@ -1,4 +1,5 @@ -using System.DirectoryServices; +# pragma warning disable CA1416 +using System.DirectoryServices; using Exceptionless.Core.Configuration; namespace Exceptionless.Core.Authentication { @@ -60,4 +61,5 @@ private SearchResult FindUser(string username) { } } } -} \ No newline at end of file +} +# pragma warning restore CA1416 \ No newline at end of file diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index d207210762..ca5e93f9f6 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -23,20 +23,20 @@ - - - - - - + + + + + + - + - + - + - + \ No newline at end of file diff --git a/src/Exceptionless.Core/Extensions/QueryNodeExtensions.cs b/src/Exceptionless.Core/Extensions/QueryNodeExtensions.cs new file mode 100644 index 0000000000..e7112ac842 --- /dev/null +++ b/src/Exceptionless.Core/Extensions/QueryNodeExtensions.cs @@ -0,0 +1,23 @@ +using System; +using Foundatio.Parsers.LuceneQueries.Nodes; + +namespace Exceptionless.Core.Extensions { + public static class QueryNodeExtensions { + public static GroupNode GetParent(this IQueryNode node, Func condition) { + if (node == null) + return null; + + IQueryNode queryNode = node; + do { + GroupNode groupNode = queryNode as GroupNode; + if (groupNode != null && condition(groupNode)) + return groupNode; + + queryNode = queryNode.Parent; + } + while (queryNode != null); + + return null; + } + } +} diff --git a/src/Exceptionless.Core/Mail/Mailer.cs b/src/Exceptionless.Core/Mail/Mailer.cs index 18ce91b600..c50eb0a70e 100644 --- a/src/Exceptionless.Core/Mail/Mailer.cs +++ b/src/Exceptionless.Core/Mail/Mailer.cs @@ -17,7 +17,7 @@ namespace Exceptionless.Core.Mail { public class Mailer : IMailer { - private readonly ConcurrentDictionary> _cachedTemplates = new ConcurrentDictionary>(); + private readonly ConcurrentDictionary> _cachedTemplates = new ConcurrentDictionary>(); private readonly IQueue _queue; private readonly FormattingPluginManager _pluginManager; private readonly AppOptions _appOptions; @@ -134,10 +134,11 @@ public Task SendOrganizationInviteAsync(User sender, Organization organization, { "InviteToken", invite.Token } }; + var body = RenderTemplate(template, data); return QueueMessageAsync(new MailMessage { To = invite.EmailAddress, Subject = subject, - Body = RenderTemplate(template, data) + Body = body }, template); } @@ -261,10 +262,11 @@ public Task SendUserPasswordResetAsync(User user) { private string RenderTemplate(string name, IDictionary data) { var template = GetCompiledTemplate(name); - return template(data); + var result = template(data); + return result?.ToString(); } - private Func GetCompiledTemplate(string name) { + private HandlebarsTemplate GetCompiledTemplate(string name) { return _cachedTemplates.GetOrAdd(name, templateName => { var assembly = typeof(Mailer).Assembly; string resourceName = $"Exceptionless.Core.Mail.Templates.{templateName}.html"; @@ -272,7 +274,8 @@ private Func GetCompiledTemplate(string name) { using (var stream = assembly.GetManifestResourceStream(resourceName)) { using (var reader = new StreamReader(stream)) { string template = reader.ReadToEnd(); - return Handlebars.Compile(template); + var compiledTemplateFunc = Handlebars.Compile(template); + return compiledTemplateFunc; } } }); diff --git a/src/Exceptionless.Core/Models/Stack.cs b/src/Exceptionless.Core/Models/Stack.cs index 055e167da2..b7465f4df1 100644 --- a/src/Exceptionless.Core/Models/Stack.cs +++ b/src/Exceptionless.Core/Models/Stack.cs @@ -8,7 +8,7 @@ using Newtonsoft.Json.Converters; namespace Exceptionless.Core.Models { - [DebuggerDisplay("Id: {Id}, Type: {Type}, Title: {Title}, TotalOccurrences: {TotalOccurrences}")] + [DebuggerDisplay("Id={Id} Type={Type} Status={Status} IsDeleted={IsDeleted} Title={Title} TotalOccurrences={TotalOccurrences}")] public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISupportSoftDeletes { public Stack() { Tags = new TagSet(); diff --git a/src/Exceptionless.Core/Models/StackSummaryModel.cs b/src/Exceptionless.Core/Models/StackSummaryModel.cs index d7afd0dd26..06f2716ebf 100644 --- a/src/Exceptionless.Core/Models/StackSummaryModel.cs +++ b/src/Exceptionless.Core/Models/StackSummaryModel.cs @@ -1,6 +1,8 @@ using System; +using System.Diagnostics; namespace Exceptionless.Core.Models { + [DebuggerDisplay("Id: {Id}, Status: {Status}, Title: {Title}, First: {FirstOccurrence}, Last: {LastOccurrence}")] public class StackSummaryModel : SummaryData { public string Id { get; set; } public string Title { get; set; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs index cda911ea16..cf169d72d3 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading.Tasks; using Exceptionless.Core.Configuration; -using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Repositories.Queries; diff --git a/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs b/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs index 783ebe553d..1c7682d4c5 100644 --- a/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs @@ -103,11 +103,11 @@ public AppFilterQueryBuilder(AppOptions options) { } var index = ctx.Options.GetElasticIndex(); - bool shouldApplyRetentionFilter = ShouldApplyRetentionFilter(index); + bool shouldApplyRetentionFilter = ShouldApplyRetentionFilter(index, ctx); string field = shouldApplyRetentionFilter ? GetDateField(index) : null; if (sfq.Stack != null) { - var stackIdFieldName = typeof(T) == typeof(Stack) ? "id" : _stackIdFieldName; + string stackIdFieldName = typeof(T) == typeof(Stack) ? "id" : _stackIdFieldName; var organization = allowedOrganizations.SingleOrDefault(o => o.Id == sfq.Stack.OrganizationId); if (organization != null) { if (shouldApplyRetentionFilter) @@ -161,13 +161,13 @@ public AppFilterQueryBuilder(AppOptions options) { return Query.DateRange(r => r.Field(field).GreaterThanOrEquals($"now/d-{(int)retentionDays}d").LessThanOrEquals("now/d+1d")); } - private bool ShouldApplyRetentionFilter(IIndex index) { + private bool ShouldApplyRetentionFilter(IIndex index, QueryBuilderContext ctx) where T : class, new() { if (index == null) throw new ArgumentNullException(nameof(index)); - + var indexType = index.GetType(); if (indexType == typeof(StackIndex)) - return true; + return !ctx.Source.IsEventStackFilterInverted(); if (indexType == typeof(EventIndex)) return true; diff --git a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs index 3e696e5e24..9f3543a596 100644 --- a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading.Tasks; using Exceptionless.Core.Extensions; @@ -6,12 +6,14 @@ using Exceptionless.Core.Repositories.Base; using Exceptionless.Core.Repositories.Options; using Foundatio.Parsers.LuceneQueries.Visitors; +using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; using Microsoft.Extensions.Logging; using Nest; using DateRange = Foundatio.Repositories.DateRange; +using Foundatio.Parsers.LuceneQueries.Nodes; namespace Exceptionless.Core.Repositories { public static class EventStackFilterQueryExtensions { @@ -21,6 +23,13 @@ public static T EnforceEventStackFilter(this T query, bool shouldEnforceEvent query.Values.Set(EnforceEventStackFilterKey, shouldEnforceEventStackFilter); return query; } + + internal const string EventStackFilterInvertedKey = "@IsStackFilterInverted"; + + public static T EventStackFilterInverted(this T query, bool eventStackFilterInverted = true) where T : IRepositoryQuery { + query.Values.Set(EventStackFilterInvertedKey, eventStackFilterInverted); + return query; + } } } @@ -29,6 +38,10 @@ public static class ReadEventStackFilterQueryExtensions { public static bool ShouldEnforceEventStackFilter(this IRepositoryQuery query) { return query.SafeGetOption(EventStackFilterQueryExtensions.EnforceEventStackFilterKey, false); } + + public static bool IsEventStackFilterInverted(this IRepositoryQuery query) { + return query.SafeGetOption(EventStackFilterQueryExtensions.EventStackFilterInvertedKey, false); + } } } @@ -38,12 +51,14 @@ public class EventStackFilterQueryBuilder : IElasticQueryBuilder { private readonly ILogger _logger; private readonly Field _inferredEventDateField; private readonly Field _inferredStackLastOccurrenceField; + private readonly EventStackFilter _eventStackFilter; public EventStackFilterQueryBuilder(IStackRepository stackRepository, ILoggerFactory loggerFactory) { _stackRepository = stackRepository; _logger = loggerFactory.CreateLogger(); _inferredEventDateField = Infer.Field(f => f.Date); _inferredStackLastOccurrenceField = Infer.Field(f => f.LastOccurrence); + _eventStackFilter = new EventStackFilter(); } public async Task BuildAsync(QueryBuilderContext ctx) where T : class, new() { @@ -60,39 +75,39 @@ public EventStackFilterQueryBuilder(IStackRepository stackRepository, ILoggerFac ctx.Source.FilterExpression(filter); } - var stackFilter = await EventStackFilterQueryVisitor.RunAsync(filter, EventStackFilterQueryMode.Stacks, ctx); - var invertedStackFilter = await EventStackFilterQueryVisitor.RunAsync(filter, EventStackFilterQueryMode.InvertedStacks, ctx); - + // when inverting to get excluded stack ids, add is_deleted as an alternate inverted criteria if (ctx.Options.GetSoftDeleteMode() == SoftDeleteQueryMode.ActiveOnly) - invertedStackFilter.Query = !String.IsNullOrEmpty(invertedStackFilter.Query) ? $"(is_deleted:true OR ({invertedStackFilter.Query}))" : "is_deleted:true"; + ctx.SetAlternateInvertedCriteria(new TermNode { Field = "is_deleted", Term = "true" }); - // queries are the same, no need to allow inverting - if (invertedStackFilter.Query == stackFilter.Query) - invertedStackFilter.IsInvertSuccessful = false; + var stackFilter = await _eventStackFilter.GetStackFilterAsync(filter, ctx); const int stackIdLimit = 10000; string[] stackIds = null; - string query = stackFilter.Query; - bool isStackIdsNegated = stackFilter.HasStatusOpen && invertedStackFilter.IsInvertSuccessful && !altInvertRequested; + string stackFilterValue = stackFilter.Filter; + bool isStackIdsNegated = stackFilter.HasStatusOpen && !altInvertRequested; if (isStackIdsNegated) - query = invertedStackFilter.Query; + stackFilterValue = stackFilter.InvertedFilter; - if (String.IsNullOrEmpty(query) && (!ctx.Source.ShouldEnforceEventStackFilter() || ctx.Options.GetSoftDeleteMode() != SoftDeleteQueryMode.ActiveOnly)) + if (String.IsNullOrEmpty(stackFilterValue) && (!ctx.Source.ShouldEnforceEventStackFilter() || ctx.Options.GetSoftDeleteMode() != SoftDeleteQueryMode.ActiveOnly)) return; - _logger.LogTrace("Stack filter: {StackFilter} Invert Success: {InvertSuccess} Inverted: {InvertedStackFilter}", stackFilter.Query, invertedStackFilter.IsInvertSuccessful, invertedStackFilter.Query); + _logger.LogTrace("Source: {Filter} Stack Filter: {StackFilter} Inverted Stack Filter: {InvertedStackFilter}", filter, stackFilter.Filter, stackFilter.InvertedFilter); if (!(ctx is IQueryVisitorContextWithValidator)) { - var systemFilterQuery = GetSystemFilterQuery(ctx); - systemFilterQuery.FilterExpression(query); + var systemFilterQuery = GetSystemFilterQuery(ctx, isStackIdsNegated); + systemFilterQuery.FilterExpression(stackFilterValue); var softDeleteMode = isStackIdsNegated ? SoftDeleteQueryMode.All : SoftDeleteQueryMode.ActiveOnly; + systemFilterQuery.EventStackFilterInverted(isStackIdsNegated); var results = await _stackRepository.GetIdsByQueryAsync(q => systemFilterQuery.As(), o => o.PageLimit(stackIdLimit).SoftDeleteMode(softDeleteMode)).AnyContext(); - if (results.Total > stackIdLimit && (isStackIdsNegated || invertedStackFilter.IsInvertSuccessful)) { + + if (results.Total > stackIdLimit && isStackIdsNegated) { + _logger.LogTrace("Query: {query} will be inverted due to id limit: {ResultCount}", stackFilterValue, results.Total); isStackIdsNegated = !isStackIdsNegated; - query = isStackIdsNegated ? invertedStackFilter.Query : stackFilter.Query; - systemFilterQuery.FilterExpression(query); + stackFilterValue = isStackIdsNegated ? stackFilter.InvertedFilter : stackFilter.Filter; + systemFilterQuery.FilterExpression(stackFilterValue); softDeleteMode = isStackIdsNegated ? SoftDeleteQueryMode.All : SoftDeleteQueryMode.ActiveOnly; + systemFilterQuery.EventStackFilterInverted(isStackIdsNegated); results = await _stackRepository.GetIdsByQueryAsync(q => systemFilterQuery.As(), o => o.PageLimit(stackIdLimit).SoftDeleteMode(softDeleteMode)).AnyContext(); } @@ -114,11 +129,12 @@ public EventStackFilterQueryBuilder(IStackRepository stackRepository, ILoggerFac ctx.Source.ExcludeStack(stackIds); } - var eventsResult = await EventStackFilterQueryVisitor.RunAsync(filter, EventStackFilterQueryMode.Events, ctx); - ctx.Source.FilterExpression(eventsResult.Query); + // Strips stack only fields and stack only special fields + var eventFilter = await _eventStackFilter.GetEventFilterAsync(filter, ctx); + ctx.Source.FilterExpression(eventFilter); } - private IRepositoryQuery GetSystemFilterQuery(IQueryVisitorContext context) { + private IRepositoryQuery GetSystemFilterQuery(IQueryVisitorContext context, bool isStackIdsNegated) { var builderContext = context as IQueryBuilderContext; var systemFilter = builderContext?.Source.GetSystemFilter(); var systemFilterQuery = systemFilter?.GetQuery().Clone(); @@ -135,6 +151,8 @@ private IRepositoryQuery GetSystemFilterQuery(IQueryVisitorContext context) { foreach (var range in systemFilterQuery.GetDateRanges() ?? Enumerable.Empty()) { if (range.Field == _inferredEventDateField || range.Field == "date") { range.Field = _inferredStackLastOccurrenceField; + if (isStackIdsNegated) // don't apply retention date filter on inverted stack queries + range.StartDate = null; range.EndDate = null; } } diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs index df0f1c8eff..fc8b95eaa8 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs @@ -4,13 +4,29 @@ using System.Threading.Tasks; using Exceptionless.Core.Extensions; using Exceptionless.Core.Repositories.Configuration; +using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; namespace Exceptionless.Core.Repositories.Queries { - public class EventStackFilterQueryVisitor : ChainableQueryVisitor { + public class EventStackFilter { + private readonly ISet _stackNonInvertedFields = new HashSet(StringComparer.OrdinalIgnoreCase) { + "organization_id", StackIndex.Alias.OrganizationId, + "project_id", StackIndex.Alias.ProjectId, + EventIndex.Alias.StackId, "stack_id", + StackIndex.Alias.Type, + }; + + private readonly ISet _stackAndEventFields = new HashSet(StringComparer.OrdinalIgnoreCase) { + "organization_id", StackIndex.Alias.OrganizationId, + "project_id", StackIndex.Alias.ProjectId, + EventIndex.Alias.StackId, "stack_id", + StackIndex.Alias.Type, + StackIndex.Alias.Tags, "tags" + }; + private readonly ISet _stackOnlyFields = new HashSet(StringComparer.OrdinalIgnoreCase) { StackIndex.Alias.LastOccurrence, "last_occurrence", StackIndex.Alias.References, "references", @@ -32,266 +48,152 @@ public class EventStackFilterQueryVisitor : ChainableQueryVisitor { StackIndex.Alias.IsHidden, "is_hidden" }; - private readonly ISet _stackNonInvertedFields = new HashSet(StringComparer.OrdinalIgnoreCase) { - "organization_id", StackIndex.Alias.OrganizationId, - "project_id", StackIndex.Alias.ProjectId, - EventIndex.Alias.StackId, "stack_id", - StackIndex.Alias.Type, - }; - - private readonly ISet _stackAndEventFields = new HashSet(StringComparer.OrdinalIgnoreCase) { - "organization_id", StackIndex.Alias.OrganizationId, - "project_id", StackIndex.Alias.ProjectId, - EventIndex.Alias.StackId, "stack_id", - StackIndex.Alias.Type, - StackIndex.Alias.Tags, "tags" - }; - - private readonly ISet _stackFields = new HashSet(StringComparer.OrdinalIgnoreCase); - - public EventStackFilterQueryVisitor(EventStackFilterQueryMode queryMode) { - _stackFields.AddRange(_stackOnlyFields); - _stackFields.AddRange(_stackAndEventFields); - - QueryMode = queryMode; + private readonly LuceneQueryParser _parser; + private readonly ChainedQueryVisitor _eventQueryVisitor; + private readonly ChainedQueryVisitor _stackQueryVisitor; + private readonly ChainedQueryVisitor _invertedStackQueryVisitor; + + public EventStackFilter() { + var stackOnlyFields = _stackOnlyFields.Union(_stackOnlySpecialFields); + var stackFields = stackOnlyFields.Union(_stackAndEventFields); + + _parser = new LuceneQueryParser(); + _eventQueryVisitor = new ChainedQueryVisitor(); + _eventQueryVisitor.AddVisitor(new RemoveFieldsQueryVisitor(stackOnlyFields)); + _eventQueryVisitor.AddVisitor(new CleanupQueryVisitor()); + + _stackQueryVisitor = new ChainedQueryVisitor(); + // remove everything not in the stack fields list + _stackQueryVisitor.AddVisitor(new RemoveFieldsQueryVisitor(f => !stackFields.Contains(f))); + _stackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); + // handles stack special fields and changing event field names to their stack equivalent + _stackQueryVisitor.AddVisitor(new StackFilterQueryVisitor()); + _stackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); + + _invertedStackQueryVisitor = new ChainedQueryVisitor(); + // remove everything not in the stack fields list + _invertedStackQueryVisitor.AddVisitor(new RemoveFieldsQueryVisitor(f => !stackFields.Contains(f))); + _invertedStackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); + // handles stack special fields and changing event field names to their stack equivalent + _invertedStackQueryVisitor.AddVisitor(new StackFilterQueryVisitor()); + _invertedStackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); + // inverts the filter + _invertedStackQueryVisitor.AddVisitor(new InvertQueryVisitor(_stackNonInvertedFields)); + _invertedStackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); } - public EventStackFilterQueryMode QueryMode { get; set; } = EventStackFilterQueryMode.Events; - public bool IsInvertSuccessful { get; set; } = true; - public bool HasStatus { get; set; } = false; - public bool HasStatusOpen { get; set; } = false; - public bool HasStackSpecificCriteria { get; set; } = false; - public bool HasStackIds { get; set; } = false; - - public override Task VisitAsync(GroupNode node, IQueryVisitorContext context) { - ApplyFilter(node, context); - - return base.VisitAsync(node, context); + public async Task GetEventFilterAsync(string query, IQueryVisitorContext context = null) { + context = context ?? new ElasticQueryVisitorContext(); + var result = await _parser.ParseAsync(query, context); + await _eventQueryVisitor.AcceptAsync(result, context); + return result.ToString(); } - public override async Task VisitAsync(TermNode node, IQueryVisitorContext context) { - var filteredNode = ApplyFilter(node, context); - if (filteredNode is GroupNode newGroupNode) { - await base.VisitAsync(newGroupNode, context); - return; - } - - if (String.Equals(node.Field, "status", StringComparison.OrdinalIgnoreCase)) { - HasStatus = true; - - if (!node.IsNegated.GetValueOrDefault() && String.Equals(node.Term, "open", StringComparison.OrdinalIgnoreCase)) - HasStatusOpen = true; - } - - if ((String.Equals(node.Field, EventIndex.Alias.StackId, StringComparison.OrdinalIgnoreCase) - || String.Equals(node.Field, "stack_id", StringComparison.OrdinalIgnoreCase)) - && !String.IsNullOrEmpty(node.Term)) { - HasStackIds = true; - } - - if (QueryMode != EventStackFilterQueryMode.InvertedStacks) - return; - - if (_stackNonInvertedFields.Contains(filteredNode.Field)) - return; + public async Task GetStackFilterAsync(string query, IQueryVisitorContext context = null) { + context = context ?? new ElasticQueryVisitorContext(); + var result = await _parser.ParseAsync(query, context); + var invertedResult = result.Clone(); - var groupNode = node.GetGroupNode(); + result = await _stackQueryVisitor.AcceptAsync(result, context); + invertedResult = await _invertedStackQueryVisitor.AcceptAsync(invertedResult, context); - // check to see if we already inverted the group - if (groupNode.Data.ContainsKey("@IsInverted")) - return; + return new StackFilter { + Filter = result.ToString(), + InvertedFilter = invertedResult.ToString(), + HasStatus = context.GetBoolean(nameof(StackFilter.HasStatus)), + HasStackIds = context.GetBoolean(nameof(StackFilter.HasStackIds)), + HasStatusOpen = context.GetBoolean(nameof(StackFilter.HasStatusOpen)) + }; + } + } - var referencedFields = await GetReferencedFieldsQueryVisitor.RunAsync(groupNode, context); - if (referencedFields.Any(f => _stackNonInvertedFields.Contains(f))) { - // if we have referenced fields that are on the list of non-inverted fields and the operator is an OR then its an issue, mark invert unsuccessful - if (node.GetOperator(context) == GroupOperator.Or) { - IsInvertSuccessful = false; - return; - } + public class StackFilterQueryVisitor : ChainableQueryVisitor { + public override Task VisitAsync(TermNode node, IQueryVisitorContext context) { + IQueryNode result = node; - node.IsNegated = node.IsNegated.HasValue ? !node.IsNegated : true; - return; + // don't include terms without fields + if (node.Field == null) { + node.RemoveSelf(); + return Task.FromResult(null); } - // negate the entire group - if (groupNode.Left != null) { - groupNode.IsNegated = groupNode.IsNegated.HasValue ? !groupNode.IsNegated : true; - if (groupNode.Right != null) - groupNode.HasParens = true; + // process special stack fields + switch (node.Field?.ToLowerInvariant()) { + case EventIndex.Alias.StackId: + case "stack_id": + node.Field = "id"; + break; + case "is_fixed": + case StackIndex.Alias.IsFixed: + bool isFixed = Boolean.TryParse(node.Term, out bool temp) && temp; + node.Field = "status"; + node.Term = "fixed"; + node.IsNegated = !isFixed; + break; + case "is_regressed": + case StackIndex.Alias.IsRegressed: + bool isRegressed = Boolean.TryParse(node.Term, out bool regressed) && regressed; + node.Field = "status"; + node.Term = "regressed"; + node.IsNegated = !isRegressed; + break; + case "is_hidden": + case StackIndex.Alias.IsHidden: + bool isHidden = Boolean.TryParse(node.Term, out bool hidden) && hidden; + if (isHidden) { + var isHiddenNode = new GroupNode { + HasParens = true, + IsNegated = true, + Operator = GroupOperator.Or, + Left = new TermNode { Field = "status", Term = "open" }, + Right = new TermNode { Field = "status", Term = "regressed" } + }; + + result = node.ReplaceSelf(isHiddenNode); + + break; + } else { + var notHiddenNode = new GroupNode { + HasParens = true, + Operator = GroupOperator.Or, + Left = new TermNode { Field = "status", Term = "open" }, + Right = new TermNode { Field = "status", Term = "regressed" } + }; + + result = node.ReplaceSelf(notHiddenNode); + + break; + } } - groupNode.Data["@IsInverted"] = true; - } + if (result is TermNode termNode) { + if (String.Equals(termNode.Field, "status", StringComparison.OrdinalIgnoreCase)) { + context.SetValue(nameof(StackFilter.HasStatus), true); - public override void Visit(TermRangeNode node, IQueryVisitorContext context) { - ApplyFilter(node, context); - } - - public override void Visit(ExistsNode node, IQueryVisitorContext context) { - ApplyFilter(node, context); - } - - public override void Visit(MissingNode node, IQueryVisitorContext context) { - ApplyFilter(node, context); - } - - private IFieldQueryNode ApplyFilter(IFieldQueryNode node, IQueryVisitorContext context) { - var parent = node.Parent as GroupNode; - - if (QueryMode == EventStackFilterQueryMode.Stacks || QueryMode == EventStackFilterQueryMode.InvertedStacks) { - // if we don't have a field name and it's a group node, leave it alone - if (node.Field == null && node is GroupNode) - return node; - - // if we have a field name and it's in the stack fields list, leave it alone - if (node.Field != null && _stackFields.Contains(node.Field)) { - if (_stackOnlyFields.Contains(node.Field)) - HasStackSpecificCriteria = true; - - return node; + if (!termNode.IsNegated.GetValueOrDefault() && String.Equals(termNode.Term, "open", StringComparison.OrdinalIgnoreCase)) + context.SetValue(nameof(StackFilter.HasStatusOpen), true); } - // check for special field names - if (node is TermNode termNode) { - switch (node.Field?.ToLowerInvariant()) { - case EventIndex.Alias.StackId: - case "stack_id": - HasStackSpecificCriteria = true; - termNode.Field = "id"; - return node; - - case "is_fixed": - case StackIndex.Alias.IsFixed: - HasStackSpecificCriteria = true; - bool isFixed = Boolean.TryParse(termNode.Term, out bool temp) && temp; - termNode.Field = "status"; - termNode.Term = "fixed"; - termNode.IsNegated = !isFixed; - return node; - - case "is_regressed": - case StackIndex.Alias.IsRegressed: - HasStackSpecificCriteria = true; - bool isRegressed = Boolean.TryParse(termNode.Term, out bool regressed) && regressed; - termNode.Field = "status"; - termNode.Term = "regressed"; - termNode.IsNegated = !isRegressed; - return node; - - case "is_hidden": - case StackIndex.Alias.IsHidden: - if (parent == null) - break; - - HasStackSpecificCriteria = true; - bool isHidden = Boolean.TryParse(termNode.Term, out bool hidden) && hidden; - if (isHidden) { - var isHiddenNode = new GroupNode { - HasParens = true, - IsNegated = true, - Operator = GroupOperator.And, - Left = new TermNode { Field = "status", Term = "open" }, - Right = new TermNode { Field = "status", Term = "regressed" } - }; - if (parent.Left == node) - parent.Left = isHiddenNode; - else if (parent.Right == node) - parent.Right = isHiddenNode; - - return isHiddenNode; - } else { - var notHiddenNode = new GroupNode { - HasParens = true, - Operator = GroupOperator.Or, - Left = new TermNode { Field = "status", Term = "open" }, - Right = new TermNode { Field = "status", Term = "regressed" } - }; - - if (parent.Left == node) - parent.Left = notHiddenNode; - else if (parent.Right == node) - parent.Right = notHiddenNode; - - return notHiddenNode; - } - } - } - - if (parent == null) - return node; - - if (parent.Left == node) - parent.Left = null; - else if (parent.Right == node) - parent.Right = null; - } else { - // don't remove terms without fields - if (String.IsNullOrEmpty(node.Field)) - return node; - - if (_stackOnlyFields.Contains(node.Field) || _stackOnlySpecialFields.Contains(node.Field)) { - // remove criteria that is only for stacks - - if (parent == null) - return node; - - if (parent.Left == node) - parent.Left = null; - else if (parent.Right == node) - parent.Right = null; + if ((String.Equals(termNode.Field, EventIndex.Alias.StackId, StringComparison.OrdinalIgnoreCase) + || String.Equals(termNode.Field, "stack_id", StringComparison.OrdinalIgnoreCase)) + && !String.IsNullOrEmpty(termNode.Term)) { + context.SetValue(nameof(StackFilter.HasStackIds), true); } } - return node; - } - - public override async Task AcceptAsync(IQueryNode node, IQueryVisitorContext context) { - await node.AcceptAsync(this, context).AnyContext(); - return node; + return Task.FromResult(result); } - public static async Task RunAsync(IQueryNode node, EventStackFilterQueryMode queryMode, IQueryVisitorContext context = null) { - var visitor = new EventStackFilterQueryVisitor(queryMode); - var stackNode = await visitor.AcceptAsync(node, context).AnyContext(); - var result = await GenerateQueryVisitor.RunAsync(stackNode, context).AnyContext(); - - return new EventStackFilterQueryResult { - Query = result, - IsInvertSuccessful = visitor.IsInvertSuccessful, - HasStatus = visitor.HasStatus, - HasStatusOpen = visitor.HasStatusOpen, - HasStackSpecificCriteria = visitor.HasStackSpecificCriteria, - HasStackIds = visitor.HasStackIds - }; - } - - public static async Task RunAsync(string query, EventStackFilterQueryMode queryMode, IQueryVisitorContext context = null) { - var parser = new LuceneQueryParser(); - var result = await parser.ParseAsync(query, context).AnyContext(); - return await RunAsync(result, queryMode, context).AnyContext(); - } - - public static EventStackFilterQueryResult Run(IQueryNode node, EventStackFilterQueryMode queryMode, IQueryVisitorContext context = null) { - return RunAsync(node, queryMode, context).GetAwaiter().GetResult(); - } - - public static EventStackFilterQueryResult Run(string query, EventStackFilterQueryMode queryMode, IQueryVisitorContext context = null) { - return RunAsync(query, queryMode, context).GetAwaiter().GetResult(); + public override Task AcceptAsync(IQueryNode node, IQueryVisitorContext context) { + return node.AcceptAsync(this, context); } } - public class EventStackFilterQueryResult { - public string Query { get; set; } - public bool IsInvertSuccessful { get; set; } + public class StackFilter { + public string Filter { get; set; } + public string InvertedFilter { get; set; } public bool HasStatus { get; set; } public bool HasStatusOpen { get; set; } - public bool HasStackSpecificCriteria { get; set; } public bool HasStackIds { get; set; } } - - public enum EventStackFilterQueryMode { - Stacks, - InvertedStacks, - Events - } } \ No newline at end of file diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs index 5f7b82e8db..4e876e746a 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs @@ -15,17 +15,17 @@ public StackDateFixedQueryVisitor(string dateFixedFieldName) { _dateFixedFieldName = dateFixedFieldName; } - public override Task VisitAsync(TermNode node, IQueryVisitorContext context) { + public override Task VisitAsync(TermNode node, IQueryVisitorContext context) { if (!String.Equals(node.Field, "fixed", StringComparison.OrdinalIgnoreCase)) - return Task.CompletedTask; + return Task.FromResult(node); if (!Boolean.TryParse(node.Term, out bool isFixed)) - return Task.CompletedTask; + return Task.FromResult(node); var query = new ExistsQuery { Field = _dateFixedFieldName }; node.SetQuery(isFixed ? query : !query); - return Task.CompletedTask; + return Task.FromResult(node); } } } \ No newline at end of file diff --git a/src/Exceptionless.Core/Validation/IsObjectIdValidator.cs b/src/Exceptionless.Core/Validation/IsObjectIdValidator.cs index af36c43c1d..2b81945e00 100644 --- a/src/Exceptionless.Core/Validation/IsObjectIdValidator.cs +++ b/src/Exceptionless.Core/Validation/IsObjectIdValidator.cs @@ -3,7 +3,9 @@ namespace Exceptionless.Core.Validation { public class IsObjectIdValidator : PropertyValidator { - public IsObjectIdValidator() : base("Value is not a valid object id.") {} + protected override string GetDefaultMessageTemplate() { + return "Value is not a valid object id."; + } protected override bool IsValid(PropertyValidatorContext context) { string value = context.PropertyValue as string; diff --git a/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj b/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj index 22883ace8f..a4ab2fb428 100644 --- a/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj +++ b/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj @@ -1,35 +1,35 @@  - - - - - + + + + + - - - - - - - + + + + + + + - + - + - - + + diff --git a/src/Exceptionless.Job/Exceptionless.Job.csproj b/src/Exceptionless.Job/Exceptionless.Job.csproj index a978e3253e..6bba99e362 100644 --- a/src/Exceptionless.Job/Exceptionless.Job.csproj +++ b/src/Exceptionless.Job/Exceptionless.Job.csproj @@ -6,15 +6,16 @@ - - - - - + + + + + + \ No newline at end of file diff --git a/src/Exceptionless.Web/Controllers/EventController.cs b/src/Exceptionless.Web/Controllers/EventController.cs index f3a115159d..e021878229 100644 --- a/src/Exceptionless.Web/Controllers/EventController.cs +++ b/src/Exceptionless.Web/Controllers/EventController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -34,6 +34,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Net.Http.Headers; using Newtonsoft.Json; +using System.Text; namespace Exceptionless.Web.Controllers { [Route(API_PREFIX + "/events")] @@ -89,17 +90,18 @@ ILoggerFactory loggerFactory /// A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value /// The time filter that limits the data being returned to a specific date range. /// The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support. + /// If no mode is set then the whole event object will be returned. If the mode is set to summary than a light weight object will be returned. /// Invalid filter. [HttpGet("count")] [Authorize(Policy = AuthorizationRoles.UserPolicy)] - public async Task> GetCountAsync(string filter = null, string aggregations = null, string time = null, string offset = null) { + public async Task> GetCountAsync(string filter = null, string aggregations = null, string time = null, string offset = null, string mode = null) { var organizations = await GetSelectedOrganizationsAsync(_organizationRepository, _projectRepository, _stackRepository, filter); - if (organizations.Count(o => !o.IsSuspended) == 0) + if (organizations.All(o => o.IsSuspended)) return Ok(CountResult.Empty); var ti = GetTimeInfo(time, offset, organizations.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetCountImplAsync(sf, ti, filter, aggregations); + return await CountInternalAsync(sf, ti, filter, aggregations, mode); } /// @@ -110,10 +112,11 @@ public async Task> GetCountAsync(string filter = null, /// A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value /// The time filter that limits the data being returned to a specific date range. /// The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support. + /// If no mode is set then the whole event object will be returned. If the mode is set to summary than a light weight object will be returned. /// Invalid filter. [HttpGet("~/" + API_PREFIX + "/organizations/{organizationId:objectid}/events/count")] [Authorize(Policy = AuthorizationRoles.UserPolicy)] - public async Task> GetCountByOrganizationAsync(string organizationId, string filter = null, string aggregations = null, string time = null, string offset = null) { + public async Task> GetCountByOrganizationAsync(string organizationId, string filter = null, string aggregations = null, string time = null, string offset = null, string mode = null) { var organization = await GetOrganizationAsync(organizationId); if (organization == null) return NotFound(); @@ -123,7 +126,7 @@ public async Task> GetCountByOrganizationAsync(string var ti = GetTimeInfo(time, offset, organization.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays)); var sf = new AppFilter(organization); - return await GetCountImplAsync(sf, ti, filter, aggregations); + return await CountInternalAsync(sf, ti, filter, aggregations, mode); } /// @@ -134,10 +137,11 @@ public async Task> GetCountByOrganizationAsync(string /// A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value /// The time filter that limits the data being returned to a specific date range. /// The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support. + /// If no mode is set then the whole event object will be returned. If the mode is set to summary than a light weight object will be returned. /// Invalid filter. [HttpGet("~/" + API_PREFIX + "/projects/{projectId:objectid}/events/count")] [Authorize(Policy = AuthorizationRoles.UserPolicy)] - public async Task> GetCountByProjectAsync(string projectId, string filter = null, string aggregations = null, string time = null, string offset = null) { + public async Task> GetCountByProjectAsync(string projectId, string filter = null, string aggregations = null, string time = null, string offset = null, string mode = null) { var project = await GetProjectAsync(projectId); if (project == null) return NotFound(); @@ -151,35 +155,7 @@ public async Task> GetCountByProjectAsync(string proje var ti = GetTimeInfo(time, offset, organization.GetRetentionUtcCutoff(project, _appOptions.MaximumRetentionDays)); var sf = new AppFilter(project, organization); - return await GetCountImplAsync(sf, ti, filter, aggregations); - } - - private async Task> GetCountImplAsync(AppFilter sf, TimeInfo ti, string filter = null, string aggregations = null) { - var pr = await _validator.ValidateQueryAsync(filter); - if (!pr.IsValid) - return BadRequest(pr.Message); - - var far = await _validator.ValidateAggregationsAsync(aggregations); - if (!far.IsValid) - return BadRequest(far.Message); - - sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || far.UsesPremiumFeatures; - var query = new RepositoryQuery() - .AppFilter(ShouldApplySystemFilter(sf, filter) ? sf : null) - .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field) - .Index(ti.Range.UtcStart, ti.Range.UtcEnd); - - CountResult result; - try { - result = await _repository.CountAsync(q => q.SystemFilter(query).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression(aggregations)); - } catch (Exception ex) { - using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Aggregations = aggregations }).Tag("Search").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext))) - _logger.LogError(ex, "An error has occurred. Please check your filter or aggregations."); - - return BadRequest("An error has occurred. Please check your search filter."); - } - - return Ok(result); + return await CountInternalAsync(sf, ti, filter, aggregations, mode); } /// @@ -234,7 +210,7 @@ public async Task> GetAsync(string id, string filt [Authorize(Policy = AuthorizationRoles.UserPolicy)] public async Task>> GetAsync(string filter = null, string sort = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10, string after = null) { var organizations = await GetSelectedOrganizationsAsync(_organizationRepository, _projectRepository, _stackRepository, filter); - if (organizations.Count(o => !o.IsSuspended) == 0) + if (organizations.All(o => o.IsSuspended)) return Ok(EmptyModels); var ti = GetTimeInfo(time, offset, organizations.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays)); @@ -242,6 +218,38 @@ public async Task>> GetAsync(s return await GetInternalAsync(sf, ti, filter, sort, mode, page, limit, after); } + private async Task> CountInternalAsync(AppFilter sf, TimeInfo ti, string filter = null, string aggregations = null, string mode = null) { + var pr = await _validator.ValidateQueryAsync(filter); + if (!pr.IsValid) + return BadRequest(pr.Message); + + var far = await _validator.ValidateAggregationsAsync(aggregations); + if (!far.IsValid) + return BadRequest(far.Message); + + sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || far.UsesPremiumFeatures; + + if (mode == "stack_new") + filter = AddFirstOccurrenceFilter(ti.Range, filter); + + var query = new RepositoryQuery() + .AppFilter(ShouldApplySystemFilter(sf, filter) ? sf : null) + .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field) + .Index(ti.Range.UtcStart, ti.Range.UtcEnd); + + CountResult result; + try { + result = await _repository.CountAsync(q => q.SystemFilter(query).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression(aggregations)); + } catch (Exception ex) { + using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Aggregations = aggregations }).Tag("Search").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext))) + _logger.LogError(ex, "An error has occurred. Please check your filter or aggregations."); + + return BadRequest("An error has occurred. Please check your search filter."); + } + + return Ok(result); + } + private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, string filter = null, string sort = null, string mode = null, int page = 1, int limit = 10, string after = null, bool usesPremiumFeatures = false) { page = GetPage(page); limit = GetLimit(limit); @@ -291,6 +299,9 @@ private async Task>> GetIntern _ => null }; + if (mode == "stack_new") + filter = AddFirstOccurrenceFilter(ti.Range, filter); + var countResponse = await _repository.CountAsync(q => q .SystemFilter(systemFilter) .FilterExpression(filter) @@ -322,6 +333,38 @@ private async Task>> GetIntern } } + private string AddFirstOccurrenceFilter(DateTimeRange timeRange, string filter) { + bool inverted = false; + if (filter != null && filter.StartsWith("@!")) { + inverted = true; + filter = filter.Substring(2); + } + + var sb = new StringBuilder(); + if (inverted) + sb.Append("@!"); + + sb.Append("first_occurrence:["); + sb.Append((long)timeRange.UtcStart.Subtract(DateTime.UnixEpoch).TotalMilliseconds); + sb.Append(" TO "); + sb.Append((long)timeRange.UtcEnd.Subtract(DateTime.UnixEpoch).TotalMilliseconds); + sb.Append(']'); + + if (String.IsNullOrEmpty(filter)) + return sb.ToString(); + + sb.Append(' '); + + bool isGrouped = filter.StartsWith('(') && filter.EndsWith(')'); + + if (isGrouped) + sb.Append(filter); + else + sb.Append('(').Append(filter).Append(')'); + + return sb.ToString(); + } + private Task> GetEventsInternalAsync(AppFilter sf, TimeInfo ti, string filter, string sort, int page, int limit, string after, bool useSearchAfter) { if (String.IsNullOrEmpty(sort)) sort = "-date"; @@ -444,7 +487,7 @@ public async Task>> GetByStack [Authorize(Policy = AuthorizationRoles.UserPolicy)] public async Task>> GetByReferenceIdAsync(string referenceId, string offset = null, string mode = null, int page = 1, int limit = 10, string after = null) { var organizations = await GetSelectedOrganizationsAsync(_organizationRepository, _projectRepository, _stackRepository); - if (organizations.Count(o => !o.IsSuspended) == 0) + if (organizations.All(o => o.IsSuspended)) return Ok(EmptyModels); var ti = GetTimeInfo(null, offset, organizations.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays)); @@ -501,7 +544,7 @@ public async Task>> GetByRefer [Authorize(Policy = AuthorizationRoles.UserPolicy)] public async Task>> GetBySessionIdAsync(string sessionId, string filter = null, string sort = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10, string after = null) { var organizations = await GetSelectedOrganizationsAsync(_organizationRepository, _projectRepository, _stackRepository, filter); - if (organizations.Count(o => !o.IsSuspended) == 0) + if (organizations.All(o => o.IsSuspended)) return Ok(EmptyModels); var ti = GetTimeInfo(time, offset, organizations.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays)); @@ -560,7 +603,7 @@ public async Task>> GetBySessi [Authorize(Policy = AuthorizationRoles.UserPolicy)] public async Task>> GetSessionsAsync(string filter = null, string sort = null, string time = null, string offset = null, string mode = null, int page = 1, int limit = 10, string after = null) { var organizations = await GetSelectedOrganizationsAsync(_organizationRepository, _projectRepository, _stackRepository, filter); - if (organizations.Count(o => !o.IsSuspended) == 0) + if (organizations.All(o => o.IsSuspended)) return Ok(EmptyModels); var ti = GetTimeInfo(time, offset, organizations.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays)); diff --git a/src/Exceptionless.Web/Exceptionless.Web.csproj b/src/Exceptionless.Web/Exceptionless.Web.csproj index c297b7a3fa..501989f305 100644 --- a/src/Exceptionless.Web/Exceptionless.Web.csproj +++ b/src/Exceptionless.Web/Exceptionless.Web.csproj @@ -1,4 +1,4 @@ - + @@ -9,15 +9,15 @@ - - + + - - - + + + - - + + diff --git a/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs b/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs index 5a147425f5..242d989530 100644 --- a/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs +++ b/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs @@ -86,7 +86,7 @@ private async Task OnDisconnected(HttpContext context, WebSocket socket, string } } - private async Task ReceiveAsync(WebSocket socket, Action handleMessage) { + private async Task ReceiveAsync(WebSocket socket, Func handleMessage) { var buffer = new ArraySegment(new byte[1024 * 4]); var result = await socket.ReceiveAsync(buffer, CancellationToken.None); LogFrame(result, buffer.Array); @@ -108,7 +108,7 @@ private async Task ReceiveAsync(WebSocket socket, Action() }, { new OpenApiSecurityScheme { Reference = new OpenApiReference { Type = ReferenceType.SecurityScheme, Id = "Bearer" } }, - new string[0] + Array.Empty() }, { new OpenApiSecurityScheme { Reference = new OpenApiReference { Type = ReferenceType.SecurityScheme, Id = "Token" } }, - new string[0] + Array.Empty() } }); diff --git a/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs b/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs index 97a187e63c..cd8048174a 100644 --- a/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs @@ -134,6 +134,38 @@ public async Task CanPostCompressedStringAsync() { var ev = (await _eventRepository.GetAllAsync()).Documents.Single(); Assert.Equal(message, ev.Message); } + + [Fact] + public async Task CanPostJsonWithUserInfoAsync() { + const string json = "{\"message\":\"test\",\"@user\":{\"identity\":\"Test user\",\"name\":null}}"; + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(json, "application/json") + .StatusCodeShouldBeAccepted() + ); + + var stats = await _eventQueue.GetQueueStatsAsync(); + Assert.Equal(1, stats.Enqueued); + Assert.Equal(0, stats.Completed); + + var processEventsJob = GetService(); + await processEventsJob.RunAsync(); + await RefreshDataAsync(); + + stats = await _eventQueue.GetQueueStatsAsync(); + Assert.Equal(1, stats.Completed); + + var events = await _eventRepository.GetAllAsync(); + var ev = events.Documents.Single(e => String.Equals(e.Type, Event.KnownTypes.Log)); + Assert.Equal("test", ev.Message); + + var userInfo = ev.GetUserIdentity(); + Assert.NotNull(userInfo); + Assert.Equal("Test user", userInfo.Identity); + Assert.Null(userInfo.Name); + } [Fact] public async Task CanPostEventAsync() { @@ -240,9 +272,9 @@ public async Task CanGetFreeProjectLevelMostFrequentStackMode() { Log.SetLogLevel(LogLevel.Trace); Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); string projectId = SampleDataService.FREE_PROJECT_ID; - var results = await SendRequestAsAsync>(r => r .AsFreeOrganizationUser() .AppendPath("projects", projectId, "events") @@ -253,18 +285,24 @@ public async Task CanGetFreeProjectLevelMostFrequentStackMode() { .StatusCodeShouldBeOk() ); - Assert.Equal(3, results.Count); + Assert.Equal(2, results.Count); } [Fact] public async Task CanGetNewStackMode() { + Log.MinimumLevel = LogLevel.Warning; await CreateStacksAndEventsAsync(); + Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); + var results = await SendRequestAsAsync>(r => r .AsGlobalAdminUser() .AppendPath("events") .QueryString("filter", $"project:{SampleDataService.TEST_PROJECT_ID} (status:open OR status:regressed)") .QueryString("mode", "stack_new") + .QueryString("time", "last 12 hours") .StatusCodeShouldBeOk() ); @@ -534,6 +572,122 @@ await CreateDataAsync(d => { Assert.Equal(1, uniqueTotal); } + [Fact] + public async Task WillExcludeOldStacksForStackNewMode() { + var utcNow = SystemClock.UtcNow; + + await CreateDataAsync(d => { + d.Event() + .TestProject() + .Message("New stack - skip due to date filter") + .Type(Event.KnownTypes.Log) + .Status(StackStatus.Open) + .TotalOccurrences(50) + .IsFirstOccurrence() + .FirstOccurrence(utcNow.SubtractYears(1)) + .LastOccurrence(utcNow.SubtractMonths(5)); + + d.Event() + .TestProject() + .Message("Old stack - new event") + .Type(Event.KnownTypes.Log) + .Status(StackStatus.Regressed) + .TotalOccurrences(33) + .FirstOccurrence(utcNow.SubtractYears(1)) + .LastOccurrence(utcNow); + + d.Event() + .TestProject() + .Message("New Stack - event not marked as first occurrence") + .Type(Event.KnownTypes.Log) + .Status(StackStatus.Open) + .TotalOccurrences(15) + .FirstOccurrence(utcNow.SubtractDays(2)) + .Version("1.2.3"); + + d.Event() + .TestProject() + .Message("New Stack - event marked as first occurrence") + .Type(Event.KnownTypes.Error) + .Status(StackStatus.Regressed) + .TotalOccurrences(10) + .FirstOccurrence(utcNow.SubtractDays(2)) + .Date(utcNow.SubtractDays(2)) + .IsFirstOccurrence() + .StackReference("https://github.com/exceptionless/Exceptionless") + .Version("3.2.1-beta1"); + + d.Event() + .TestProject() + .Message("Deleted New stack - event is first occurrence") + .Type(Event.KnownTypes.FeatureUsage) + .Status(StackStatus.Open) + .TotalOccurrences(7) + .FirstOccurrence(utcNow.Date) + .IsFirstOccurrence() + .Date(utcNow.Date) + .Deleted(); + }); + + Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); + + const string filter = "(status:open OR status:regressed)"; + const string time = "last week"; + + /* + _logger.LogInformation("Running non-inverted query"); + var invertedResults = await SendRequestAsAsync>(r => r + .AsGlobalAdminUser() + .AppendPath("events") + .QueryString("filter", "@!" + filter) + .QueryString("time", time) + .QueryString("mode", "stack_new") + .StatusCodeShouldBeOk() + );*/ + + //Assert.Equal(2, invertedResults.Count); + + _logger.LogInformation("Running inverted query"); + var results = await SendRequestAsAsync>(r => r + .AsGlobalAdminUser() + .AppendPath("events") + .QueryString("filter", filter) + .QueryString("time", time) + .QueryString("mode", "stack_new") + .StatusCodeShouldBeOk() + ); + + Assert.Equal(2, results.Count); + + _logger.LogInformation("Running normal count"); + var countResult = await SendRequestAsAsync(r => r + .AsGlobalAdminUser() + .AppendPath("events", "count") + .QueryString("filter", filter) + .QueryString("time", time) + .QueryString("mode", "stack_new") + .QueryString("aggregations", "date:(date cardinality:stack sum:count~1) cardinality:stack terms:(first @include:true) sum:count~1") + .StatusCodeShouldBeOk() + ); + + var dateAgg = countResult.Aggregations.DateHistogram("date_date"); + double dateAggStackCount = dateAgg.Buckets.Sum(t => t.Aggregations.Cardinality("cardinality_stack").Value.GetValueOrDefault()); + double dateAggEventCount = dateAgg.Buckets.Sum(t => t.Aggregations.Cardinality("sum_count").Value.GetValueOrDefault()); + Assert.Equal(2, dateAggStackCount); + Assert.Equal(2, dateAggEventCount); + + var total = countResult.Aggregations.Sum("sum_count")?.Value; + double newTotal = countResult.Aggregations.Terms("terms_first")?.Buckets.FirstOrDefault()?.Total ?? 0; + double uniqueTotal = countResult.Aggregations.Cardinality("cardinality_stack")?.Value ?? 0; + + Assert.Equal(2, total); + Assert.Equal(1, newTotal); + Assert.Equal(2, uniqueTotal); + } + private async Task CreateStacksAndEventsAsync() { var utcNow = SystemClock.UtcNow; @@ -563,7 +717,7 @@ await CreateDataAsync(d => { .Type(Event.KnownTypes.Error) .Status(StackStatus.Regressed) .TotalOccurrences(50) - .FirstOccurrence(utcNow.SubtractDays(1)) + .FirstOccurrence(utcNow.SubtractDays(2)) .StackReference("https://github.com/exceptionless/Exceptionless") .Tag("Blake Niemyjski") .RequestInfoSample() @@ -584,8 +738,10 @@ await CreateDataAsync(d => { d.Event().FreeProject(); }); + Log.MinimumLevel = LogLevel.Warning; await StackData.CreateSearchDataAsync(GetService(), GetService(), true); await EventData.CreateSearchDataAsync(GetService(), _eventRepository, GetService(), true); + Log.MinimumLevel = LogLevel.Trace; } } } diff --git a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj index 85269c4824..faff6b3db6 100644 --- a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj +++ b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj @@ -5,11 +5,11 @@ - - - - - + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Exceptionless.Tests/Extensions/TestServerExtensions.cs b/tests/Exceptionless.Tests/Extensions/TestServerExtensions.cs index 8e338d81cf..76f112a244 100644 --- a/tests/Exceptionless.Tests/Extensions/TestServerExtensions.cs +++ b/tests/Exceptionless.Tests/Extensions/TestServerExtensions.cs @@ -7,10 +7,16 @@ namespace Exceptionless.Tests { public static class TestServerExtensions { - public static async Task WaitForReadyAsync(this TestServer server, TimeSpan? maxWaitTime = null) { + private static bool _alreadyWaited; + + public static async Task WaitForReadyAsync(this TestServer server) { var startupContext = server.Services.GetService(); - maxWaitTime ??= Debugger.IsAttached ? TimeSpan.FromMinutes(5) :TimeSpan.FromSeconds(5); - + var maxWaitTime = !_alreadyWaited ? TimeSpan.FromSeconds(30) : TimeSpan.FromSeconds(2); + if (Debugger.IsAttached) + maxWaitTime = maxWaitTime.Add(TimeSpan.FromMinutes(1)); + + _alreadyWaited = true; + var client = server.CreateClient(); var startTime = DateTime.Now; do { diff --git a/tests/Exceptionless.Tests/IntegrationTestsBase.cs b/tests/Exceptionless.Tests/IntegrationTestsBase.cs index ae34f9e786..24df55bac6 100644 --- a/tests/Exceptionless.Tests/IntegrationTestsBase.cs +++ b/tests/Exceptionless.Tests/IntegrationTestsBase.cs @@ -102,13 +102,13 @@ protected virtual void RegisterServices(IServiceCollection services) { services.AddSingleton(); services.AddSingleton(); - services.AddTransient(); + services.AddTransient(); services.ReplaceSingleton(s => _server.CreateHandler()); } public async Task<(List Stacks, List Events)> CreateDataAsync(Action dataBuilderFunc) { - var eventBuilders = new List(); + var eventBuilders = new List(); var dataBuilder = new DataBuilder(eventBuilders, ServiceProvider); dataBuilderFunc(dataBuilder); @@ -121,7 +121,7 @@ protected virtual void RegisterServices(IServiceCollection services) { foreach (var builder in eventBuilders) { var data = builder.Build(); - events.Add(data.Event); + events.AddRange(data.Events); stacks.Add(data.Stack); } diff --git a/tests/Exceptionless.Tests/Mail/MailerTests.cs b/tests/Exceptionless.Tests/Mail/MailerTests.cs index e5d801ab99..24d2a6c3d0 100644 --- a/tests/Exceptionless.Tests/Mail/MailerTests.cs +++ b/tests/Exceptionless.Tests/Mail/MailerTests.cs @@ -199,8 +199,10 @@ public async Task SendOrganizationInviteAsync() { await RunMailJobAsync(); - if (GetService() is InMemoryMailSender sender) - Assert.Contains("Join Organization", sender.LastMessage.Body); + var sender = GetService() as InMemoryMailSender; + Assert.NotNull(sender); + + Assert.Contains("Join Organization", sender.LastMessage.Body); } [Fact] diff --git a/tests/Exceptionless.Tests/Search/EventStackFilterQueryTests.cs b/tests/Exceptionless.Tests/Search/EventStackFilterQueryTests.cs new file mode 100644 index 0000000000..8fed2b78ef --- /dev/null +++ b/tests/Exceptionless.Tests/Search/EventStackFilterQueryTests.cs @@ -0,0 +1,113 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Queries; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Utility; +using Foundatio.Parsers.ElasticQueries.Visitors; +using Foundatio.Repositories; +using Microsoft.Extensions.Logging; +using Xunit; +using Xunit.Abstractions; + +namespace Exceptionless.Tests.Search { + public class EventStackFilterQueryTests : IntegrationTestsBase { + private readonly IStackRepository _stackRepository; + private readonly IEventRepository _eventRepository; + private static bool _isTestDataGenerated; + + public EventStackFilterQueryTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { + _stackRepository = GetService(); + _eventRepository = GetService(); + } + + protected override async Task ResetDataAsync() { + if (_isTestDataGenerated) + return; + + await base.ResetDataAsync(); + await CreateDataAsync(d => { + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Open).Source("Namespace.ClassName").UserIdentity("test@test.com", "Test Test"); + d.Event().Type(Event.KnownTypes.Log).FreeProject().Status(StackStatus.Open); + d.Event().StackId(TestConstants.StackId).Type(Event.KnownTypes.Log).Status(StackStatus.Open); + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Open).Deleted(); + + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Fixed); + + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Ignored); + d.Event().Type(Event.KnownTypes.Session).Status(StackStatus.Ignored); + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Open).ReferenceId("referenceId"); + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Open).SessionId("sessionId"); + + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Discarded); + d.Event().Type(Event.KnownTypes.Log).Status(StackStatus.Regressed); + }); + + _isTestDataGenerated = true; + } + + [Theory] + [InlineData("status:open OR status:regressed", 7)] + [InlineData("NOT (status:open OR status:regressed)", 4)] + [InlineData("status:fixed", 1)] + [InlineData("NOT status:fixed", 10)] + [InlineData("stack:" + TestConstants.StackId, 1)] + [InlineData("stack_id:" + TestConstants.StackId, 1)] + [InlineData("-stack:" + TestConstants.StackId, 10)] + [InlineData("stack:" + TestConstants.StackId + " (status:open OR status:regressed)", 1)] + [InlineData("is_fixed:true", 1)] + [InlineData("is_regressed:true", 1)] + [InlineData("is_hidden:true", 4)] + [InlineData("project:" + SampleDataService.TEST_PROJECT_ID + " (status:open OR status:regressed)", 6, 4)] + public async Task VerifyStackFilter(string filter, int expected, int? expectedInverted = null) { + Log.SetLogLevel(LogLevel.Trace); + + var totalStacks = await _stackRepository.CountAsync(o => o.IncludeSoftDeletes()); + + var ctx = new ElasticQueryVisitorContext(); + var stackFilter = await new EventStackFilter().GetStackFilterAsync(filter, ctx); + _logger.LogInformation("Finding Filter: {Filter}", stackFilter.Filter); + var stacks = await _stackRepository.FindAsync(q => q.FilterExpression(stackFilter.Filter), o => o.SoftDeleteMode(SoftDeleteQueryMode.All).PageLimit(1000)); + Assert.Equal(expected, stacks.Total); + + _logger.LogInformation("Finding Inverted Filter: {Filter}", stackFilter.InvertedFilter); + var invertedStacks = await _stackRepository.FindAsync(q => q.FilterExpression(stackFilter.InvertedFilter), o => o.SoftDeleteMode(SoftDeleteQueryMode.All).PageLimit(1000)); + var expectedInvert = expectedInverted.HasValue ? expectedInverted.Value : totalStacks - expected; + Assert.Equal(expectedInvert, invertedStacks.Total); + + var stackIds = new HashSet(stacks.Hits.Select(h => h.Id)); + var invertedStackIds = new HashSet(invertedStacks.Hits.Select(h => h.Id)); + + Assert.Empty(stackIds.Intersect(invertedStackIds)); + } + + [Theory] + [InlineData("status:open OR status:regressed", 6)] + [InlineData("NOT (status:open OR status:regressed)", 4)] + [InlineData("status:fixed", 1)] + [InlineData("NOT status:fixed", 9)] + [InlineData("stack:" + TestConstants.StackId, 1)] + [InlineData("stack_id:" + TestConstants.StackId, 1)] + [InlineData("-stack:" + TestConstants.StackId, 9)] + [InlineData("stack:" + TestConstants.StackId + " (status:open OR status:regressed)", 1)] + [InlineData("is_fixed:true", 1)] + [InlineData("is_regressed:true", 1)] + [InlineData("is_hidden:true", 4)] + [InlineData("project:" + SampleDataService.TEST_PROJECT_ID + " (status:open OR status:regressed)", 5)] + [InlineData("project:" + SampleDataService.TEST_PROJECT_ID + " (status:open OR status:regressed) ref.session:sessionId", 1)] + [InlineData("project:" + SampleDataService.TEST_PROJECT_ID + " (status:open OR status:regressed) reference:referenceId", 1)] + public async Task VerifyEventFilter(string filter, int expected) { + Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); + Log.SetLogLevel(LogLevel.Trace); + + var events = await _eventRepository.FindAsync(q => q.FilterExpression(filter), o => o.PageLimit(1000)); + Assert.Equal(expected, events.Total); + + var invertedEvents = await _eventRepository.FindAsync(q => q.FilterExpression("@!" + filter), o => o.PageLimit(1000)); + Assert.Equal(expected, invertedEvents.Total); + } + } +} \ No newline at end of file diff --git a/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs b/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs index 43ad9b2c80..d89ffb1bc0 100644 --- a/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs +++ b/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs @@ -1,4 +1,8 @@ -using Exceptionless.Core.Repositories.Queries; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Exceptionless.Core.Repositories.Queries; +using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; @@ -7,29 +11,146 @@ public class EventStackFilterQueryVisitorTests : TestWithServices { public EventStackFilterQueryVisitorTests(ITestOutputHelper output) : base(output) { } [Theory] - [InlineData("blah", "", "", "blah")] - [InlineData("status:fixed", "status:fixed", "NOT status:fixed", "")] - [InlineData("is_fixed:true", "status:fixed", "NOT status:fixed", "")] - [InlineData("is_regressed:true", "status:regressed", "NOT status:regressed", "")] - [InlineData("is_hidden:true", "NOT (status:open AND status:regressed)", "(status:open AND status:regressed)", "")] - [InlineData("is_hidden:false", "(status:open OR status:regressed)", "NOT (status:open OR status:regressed)", "")] - [InlineData("blah:true (status:fixed OR status:open)", "(status:fixed OR status:open)", "NOT (status:fixed OR status:open)", "blah:true")] - [InlineData("blah:true", "", "", "blah:true")] - [InlineData("type:session", "type:session", "type:session", "type:session")] - [InlineData("(organization:123 AND type:log) AND (blah:true (status:fixed OR status:open))", "(organization:123 AND type:log) AND ((status:fixed OR status:open))", "(organization:123 AND type:log) AND (NOT (status:fixed OR status:open))", "(organization:123 AND type:log) AND (blah:true )")] - [InlineData("project:123 (status:open OR status:regressed) (ref.session:5f3dce2668de920001466635)", "project:123 (status:open OR status:regressed)", "project:123 NOT (status:open OR status:regressed)", "project:123 (ref.session:5f3dce2668de920001466635)")] - [InlineData("project:123 (status:open OR status:regressed) (ref.session:5f3dce2668de920001466635 OR project:234)", "project:123 (status:open OR status:regressed) (project:234)", "project:123 NOT (status:open OR status:regressed) (project:234)", "project:123 (ref.session:5f3dce2668de920001466635 OR project:234)")] - public void GetStackQuery(string filter, string expectedStackFilter, string expectedInvertedStackFilter, string expectedEventFilter) { - Log.SetLogLevel(Microsoft.Extensions.Logging.LogLevel.Trace); - - var stackResult = EventStackFilterQueryVisitor.Run(filter, EventStackFilterQueryMode.Stacks); - Assert.Equal(expectedStackFilter, stackResult.Query.Trim()); - - var invertedStackResult = EventStackFilterQueryVisitor.Run(filter, EventStackFilterQueryMode.InvertedStacks); - Assert.Equal(expectedInvertedStackFilter, invertedStackResult.Query.Trim()); - - var eventResult = EventStackFilterQueryVisitor.Run(filter, EventStackFilterQueryMode.Events); - Assert.Equal(expectedEventFilter, eventResult.Query.Trim()); + [MemberData(nameof(FilterData.TestCases), MemberType = typeof(FilterData))] + public async Task CanBuildStackFilter(FilterScenario scenario) { + Log.SetLogLevel(Microsoft.Extensions.Logging.LogLevel.Trace); + + var eventStackFilter = new EventStackFilter(); + var stackFilter = await eventStackFilter.GetStackFilterAsync(scenario.Source); + Assert.Equal(scenario.Stack, stackFilter.Filter.Trim()); + } + + [Theory] + [MemberData(nameof(FilterData.TestCases), MemberType = typeof(FilterData))] + public async Task CanBuildInvertedStackFilter(FilterScenario scenario) { + Log.SetLogLevel(Microsoft.Extensions.Logging.LogLevel.Trace); + + var eventStackFilter = new EventStackFilter(); + var stackFilter = await eventStackFilter.GetStackFilterAsync(scenario.Source); + Assert.Equal(scenario.InvertedStack, stackFilter.InvertedFilter.Trim()); + } + + [Theory] + [MemberData(nameof(FilterData.TestCases), MemberType = typeof(FilterData))] + public async Task CanBuildEventFilter(FilterScenario scenario) { + Log.SetLogLevel(Microsoft.Extensions.Logging.LogLevel.Trace); + + var eventStackFilter = new EventStackFilter(); + var stackFilter = await eventStackFilter.GetEventFilterAsync(scenario.Source); + Assert.Equal(scenario.Event, stackFilter.Trim()); + } + } + + public class FilterData { + public static IEnumerable TestCases() { + yield return new object[] { new FilterScenario { + Source = "blah", + Stack = "", + InvertedStack = "", + Event = "blah" + }}; + yield return new object[] { new FilterScenario { + Source = "status:fixed", + Stack = "status:fixed", + InvertedStack = "NOT status:fixed", + Event = "" + }}; + yield return new object[] { new FilterScenario { + Source = "is_fixed:true", + Stack = "status:fixed", + InvertedStack = "NOT status:fixed", + Event = "" + }}; + yield return new object[] { new FilterScenario { + Source = "is_regressed:true", + Stack = "status:regressed", + InvertedStack = "NOT status:regressed", + Event = "" + }}; + yield return new object[] { new FilterScenario { + Source = "is_hidden:true", + Stack = "NOT (status:open OR status:regressed)", + InvertedStack = "(status:open OR status:regressed)", + Event = "" + }}; + yield return new object[] { new FilterScenario { + Source = "is_hidden:false", + Stack = "(status:open OR status:regressed)", + InvertedStack = "NOT (status:open OR status:regressed)", + Event = "" + }}; + yield return new object[] { new FilterScenario { + Source = "blah:true (status:fixed OR status:open)", + Stack = "(status:fixed OR status:open)", + InvertedStack = "NOT (status:fixed OR status:open)", + Event = "blah:true" + }}; + yield return new object[] { new FilterScenario { + Source = "blah:true", + Stack = "", + InvertedStack = "", + Event = "blah:true" + }}; + yield return new object[] { new FilterScenario { + Source = "type:session", + Stack = "type:session", + InvertedStack = "type:session", + Event = "type:session" + }}; + yield return new object[] { new FilterScenario { + Source = "(organization:123 AND type:log) AND (blah:true (status:fixed OR status:open))", + Stack = "(organization:123 AND type:log) AND (status:fixed OR status:open)", + InvertedStack = "(organization:123 AND type:log) AND NOT (status:fixed OR status:open)", + Event = "(organization:123 AND type:log) AND blah:true" + }}; + yield return new object[] { new FilterScenario { + Source = "project:123 (status:open OR status:regressed) (ref.session:5f3dce2668de920001466635)", + Stack = "project:123 (status:open OR status:regressed)", + InvertedStack = "project:123 NOT (status:open OR status:regressed)", + Event = "project:123 ref.session:5f3dce2668de920001466635" + }}; + yield return new object[] { new FilterScenario { + Source = "project:123 (status:open OR status:regressed) (ref.session:5f3dce2668de920001466635 OR project:234)", + Stack = "project:123 (status:open OR status:regressed) project:234", + InvertedStack = "project:123 NOT (status:open OR status:regressed) project:234", + Event = "project:123 (ref.session:5f3dce2668de920001466635 OR project:234)" + }}; + yield return new object[] { new FilterScenario { + Source = "first_occurrence:[1608854400000 TO 1609188757249] AND (status:open OR status:regressed)", + Stack = "first_occurrence:[1608854400000 TO 1609188757249] AND (status:open OR status:regressed)", + InvertedStack = "NOT (first_occurrence:[1608854400000 TO 1609188757249] AND (status:open OR status:regressed))", + Event = "" + }}; + yield return new object[] { new FilterScenario { + Source = "project:537650f3b77efe23a47914f4 first_occurrence:[1609459200000 TO 1609730450521] (status:open OR status:regressed)", + Stack = "project:537650f3b77efe23a47914f4 first_occurrence:[1609459200000 TO 1609730450521] (status:open OR status:regressed)", + InvertedStack = "project:537650f3b77efe23a47914f4 NOT (first_occurrence:[1609459200000 TO 1609730450521] (status:open OR status:regressed))", + Event = "project:537650f3b77efe23a47914f4" + }}; + } + } + + public class FilterScenario : IXunitSerializable { + public string Source { get; set; } = String.Empty; + public string Stack { get; set; } = String.Empty; + public string InvertedStack { get; set; } = String.Empty; + public string Event { get; set; } = String.Empty; + + public override string ToString() { + return $"Source: \"{Source}\" Stack: \"{Stack}\" InvertedStack: \"{InvertedStack}\" Event: \"{Event}\""; + } + + public void Deserialize(IXunitSerializationInfo info) { + var value = JsonConvert.DeserializeObject(info.GetValue("objValue")); + Source = value.Source; + Stack = value.Stack; + InvertedStack = value.InvertedStack; + Event = value.Event; + } + + public void Serialize(IXunitSerializationInfo info) { + var json = JsonConvert.SerializeObject(this); + info.AddValue("objValue", json); } } } diff --git a/tests/Exceptionless.Tests/Search/MoreEventIndexTests.cs b/tests/Exceptionless.Tests/Search/MoreEventIndexTests.cs index 8c0f959924..79c656866c 100644 --- a/tests/Exceptionless.Tests/Search/MoreEventIndexTests.cs +++ b/tests/Exceptionless.Tests/Search/MoreEventIndexTests.cs @@ -38,7 +38,7 @@ public MoreEventIndexTests(ITestOutputHelper output, AppWebHostFactory factory) [InlineData("source:GET", 2)] [InlineData("source:gEt", 2)] [InlineData("source:Print", 2)] - [InlineData("source:/Print", 1)] + [InlineData("source:\"/Print\"", 1)] [InlineData("source:Bagle", 1)] [InlineData("source:exceptionless.web*", 1)] [InlineData("source:reason", 1)] diff --git a/tests/Exceptionless.Tests/Utility/DataBuilder.cs b/tests/Exceptionless.Tests/Utility/DataBuilder.cs index ac80f14a24..078c033693 100644 --- a/tests/Exceptionless.Tests/Utility/DataBuilder.cs +++ b/tests/Exceptionless.Tests/Utility/DataBuilder.cs @@ -6,6 +6,7 @@ using Exceptionless.Core.Models.Data; using Exceptionless.Core.Plugins.Formatting; using Exceptionless.Core.Utility; +using Exceptionless.DateTimeExtensions; using Exceptionless.Extensions; using Foundatio.Repositories.Utility; using Foundatio.Serializer; @@ -32,7 +33,8 @@ public class EventDataBuilder { private readonly FormattingPluginManager _formattingPluginManager; private readonly ISerializer _serializer; private readonly ICollection> _stackMutations; - private PersistentEvent _event = new PersistentEvent(); + private int _additionalEventsToCreate = 0; + private readonly PersistentEvent _event = new PersistentEvent(); private Stack _stack = null; private EventDataBuilder _stackEventBuilder; private bool _isFirstOccurrenceSet = false; @@ -76,7 +78,6 @@ public EventDataBuilder StackId(string stackId) { public EventDataBuilder Id(string id) { _event.Id = id; - return this; } @@ -121,11 +122,9 @@ public EventDataBuilder Date(DateTime date) { public EventDataBuilder Date(string date) { if (DateTimeOffset.TryParse(date, out var dt)) - _event.Date = dt; - else - throw new ArgumentException("Invalid date specified", nameof(date)); - - return this; + return Date(dt); + + throw new ArgumentException("Invalid date specified", nameof(date)); } public EventDataBuilder IsFirstOccurrence(bool isFirstOccurrence = true) { @@ -142,15 +141,14 @@ public EventDataBuilder CreatedDate(DateTime createdUtc) { public EventDataBuilder CreatedDate(string createdUtc) { if (DateTime.TryParse(createdUtc, out var dt)) - _event.CreatedUtc = dt; - else - throw new ArgumentException("Invalid date specified", nameof(createdUtc)); - - return this; + return CreatedDate(dt); + + throw new ArgumentException("Invalid date specified", nameof(createdUtc)); } public EventDataBuilder Message(string message) { _event.Message = message; + _stackMutations.Add(s => s.Title = message); return this; } @@ -269,7 +267,7 @@ public EventDataBuilder Deleted() { } public EventDataBuilder Status(StackStatus status) { - _stackMutations.Add(s => s.Status = StackStatus.Open); + _stackMutations.Add(s => s.Status = status); return this; } @@ -284,8 +282,10 @@ public EventDataBuilder OccurrencesAreCritical(bool occurrencesAreCritical = tru if (occurrencesAreCritical) _event.MarkAsCritical(); - _stackMutations.Add(s => s.OccurrencesAreCritical = occurrencesAreCritical); - + _stackMutations.Add(s => { + s.OccurrencesAreCritical = occurrencesAreCritical; + s.Tags.Add(Event.KnownTags.Critical); + }); return this; } @@ -294,8 +294,19 @@ public EventDataBuilder TotalOccurrences(int totalOccurrences) { return this; } + + public EventDataBuilder Create(int additionalOccurrences) { + _additionalEventsToCreate = additionalOccurrences; + _stackMutations.Add(s => { + if (s.TotalOccurrences <= additionalOccurrences) + s.TotalOccurrences = additionalOccurrences + 1; + }); + return this; + } + public EventDataBuilder FirstOccurrence(DateTime firstOccurrenceUtc) { + _event.CreatedUtc = firstOccurrenceUtc; _stackMutations.Add(s => s.FirstOccurrence = firstOccurrenceUtc); return this; @@ -303,49 +314,53 @@ public EventDataBuilder FirstOccurrence(DateTime firstOccurrenceUtc) { public EventDataBuilder FirstOccurrence(string firstOccurrenceUtc) { if (DateTime.TryParse(firstOccurrenceUtc, out var dt)) - _event.CreatedUtc = dt; - else - throw new ArgumentException("Invalid date specified", nameof(firstOccurrenceUtc)); - - _stackMutations.Add(s => s.FirstOccurrence = dt); - - return this; + return FirstOccurrence(dt); + + throw new ArgumentException("Invalid date specified", nameof(firstOccurrenceUtc)); } public EventDataBuilder LastOccurrence(DateTime lastOccurrenceUtc) { - _stackMutations.Add(s => s.LastOccurrence = lastOccurrenceUtc); + if (_event.CreatedUtc.IsAfter(lastOccurrenceUtc)) + _event.CreatedUtc = lastOccurrenceUtc; + if (_event.Date.IsAfter(lastOccurrenceUtc)) + _event.Date = lastOccurrenceUtc; + + _stackMutations.Add(s => { + if (s.FirstOccurrence.IsAfter(lastOccurrenceUtc)) + s.FirstOccurrence = lastOccurrenceUtc; + + s.LastOccurrence = lastOccurrenceUtc; + }); + return this; } public EventDataBuilder LastOccurrence(string lastOccurrenceUtc) { if (DateTime.TryParse(lastOccurrenceUtc, out var dt)) - _event.CreatedUtc = dt; - else - throw new ArgumentException("Invalid date specified", nameof(lastOccurrenceUtc)); + return LastOccurrence(dt); - _stackMutations.Add(s => s.LastOccurrence = dt); - - return this; + throw new ArgumentException("Invalid date specified", nameof(lastOccurrenceUtc)); } public EventDataBuilder DateFixed(DateTime? dateFixed = null) { Status(StackStatus.Fixed); - _stackMutations.Add(s => s.DateFixed = dateFixed ?? SystemClock.UtcNow); + _stackMutations.Add(s => { + var fixedOn = dateFixed ?? SystemClock.UtcNow; + if (s.FirstOccurrence.IsAfter(fixedOn)) + throw new ArgumentException("Fixed on date is before first occurence"); + + s.DateFixed = fixedOn; + }); return this; } public EventDataBuilder DateFixed(string dateFixedUtc) { if (DateTime.TryParse(dateFixedUtc, out var dt)) - _event.CreatedUtc = dt; - else - throw new ArgumentException("Invalid date specified", nameof(dateFixedUtc)); - - Status(StackStatus.Fixed); - _stackMutations.Add(s => s.DateFixed = dt); - - return this; + return DateFixed(dt); + + throw new ArgumentException("Invalid date specified", nameof(dateFixedUtc)); } public EventDataBuilder FixedInVersion(string version) { @@ -368,9 +383,9 @@ public Stack GetStack() { } private bool _isBuilt = false; - public (Stack Stack, PersistentEvent Event) Build() { - if (_isBuilt) - return (_stack, _event); + public (Stack Stack, PersistentEvent[] Events) Build() { + if (_isBuilt) + return (_stack, BuildEvents(_stack, _event)); if (String.IsNullOrEmpty(_event.OrganizationId)) _event.OrganizationId = SampleDataService.TEST_ORG_ID; @@ -464,8 +479,25 @@ public Stack GetStack() { _event.StackId = _stack.Id; _isBuilt = true; + return (_stack, BuildEvents(_stack, _event)); + } + + private PersistentEvent[] BuildEvents(Stack stack, PersistentEvent ev) { + var events = new List(_additionalEventsToCreate) { ev }; + if (_additionalEventsToCreate <= 0) + return events.ToArray(); + + int interval = (stack.LastOccurrence - stack.FirstOccurrence).Milliseconds / _additionalEventsToCreate; + for (int index = 0; index < stack.TotalOccurrences - 1; index++) { + var clone = ev.DeepClone(); + clone.Id = null; + if (interval > 0) + clone.Date = new DateTimeOffset(stack.FirstOccurrence.AddMilliseconds(interval * index), ev.Date.Offset); + + events.Add(clone); + } - return (_stack, _event); + return events.ToArray(); } private const string _sampleRequestInfo = @"{