Stack filtering improvements and bug fixes (#825)

* #746 - Add more test coverage for resolving new stack counts

* Fixed some bugs with test data builder

* Update deps. Some k8s updates.

* Progress on new stack filter issue.

* More progress

* Don't apply retention filter to stack id filter when inverted

* Got the main test passing. Not sure if the other failures are correct or not.

* Fixed one failing unit test

* Some minor changes

* Disable AD Windows build warnings #493

* Added ability to generate many events using TotalOccurrences

* Refactored how additional events are created.

* Added new test for posting null session identity name

* WIP - Event Stack Filter Tests

* Update Deps

* Fixed some build messages

* Fix issue with message bus broker async fire and forget.

* Working on stack inverting issues

* Update ES docker to 7.12

* Progress in stack filter refactor

* Fix a couple tests

* Fixing more tests

* Fix remaining tests. Update repos.

* Remove repos and parser projects

* Update deps / respond to feedback

Co-authored-by: Blake Niemyjski <[email protected]>

.github/workflows/build.yaml

@@ -19,7 +19,7 @@ jobs:
       run: "echo ref: ${{github.ref}} event: ${{github.event_name}}"
     - name: Build Version
       run: |
-        dotnet tool install --global minver-cli --version 2.3.1
+        dotnet tool install --global minver-cli --version 2.5.0
         version=$(minver --tag-prefix v)
         echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV
         echo "VERSION=$version" >> $GITHUB_ENV

.gitignore

@@ -126,3 +126,5 @@ k8s/ex-prod\.yaml
 *secrets*
 k8s/ex-*-snapshots.yaml
 node_modules
+
+*.DotSettings

Dockerfile

@@ -78,7 +78,7 @@ CMD [ "dotnet", "Exceptionless.Web.dll" ]
 
 # completely self-contained
 
-FROM exceptionless/elasticsearch:7.10.0 AS exceptionless
+FROM exceptionless/elasticsearch:7.12.0 AS exceptionless
 
 WORKDIR /app
 COPY --from=api-publish /app/src/Exceptionless.Web/out ./

Exceptionless.sln

@@ -13,6 +13,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		CONTRIBUTING.md = CONTRIBUTING.md
 		docker-compose.yml = docker-compose.yml
 		Dockerfile = Dockerfile
+		build\docker\elasticsearch\Dockerfile = build\docker\elasticsearch\Dockerfile
 		global.json = global.json
 		LICENSE.txt = LICENSE.txt
 		NuGet.Config = NuGet.Config
@@ -58,7 +59,6 @@ Global
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
-			GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {1A90AFA5-B81C-4B1B-9DFA-2D90F8CA0EF0}
 	EndGlobalSection
 EndGlobal

build/common.props

@@ -5,7 +5,7 @@
     <Product>Exceptionless</Product>
     <MinVerSkip Condition="'$(Configuration)' == 'Debug'">true</MinVerSkip>
     <MinVerTagPrefix>v</MinVerTagPrefix>
-    <Copyright>Copyright (c) 2020 Exceptionless.  All rights reserved.</Copyright>
+    <Copyright>Copyright (c) 2021 Exceptionless.  All rights reserved.</Copyright>
     <RepositoryUrl>https://github.com/exceptionless/exceptionless</RepositoryUrl>
     <Authors>Exceptionless</Authors>
     <NoWarn>$(NoWarn);CS1591</NoWarn>
@@ -19,8 +19,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All"/>
-    <PackageReference Include="AsyncFixer" Version="1.3.0" PrivateAssets="All" />
-    <PackageReference Include="MinVer" Version="2.3.1" PrivateAssets="All" />
+    <PackageReference Include="AsyncFixer" Version="1.5.1" PrivateAssets="All" />
+    <PackageReference Include="MinVer" Version="2.5.0" PrivateAssets="All" />
   </ItemGroup>
 
 </Project>

build/docker/elasticsearch/Dockerfile

@@ -1,5 +1,5 @@
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/elasticsearch/elasticsearch:7.11.0
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.12.0
 
 RUN elasticsearch-plugin install -b mapper-size
 RUN elasticsearch-plugin install -b repository-azure

docker-compose.yml

@@ -5,7 +5,7 @@ version: '3.5'
 
 services:
   elasticsearch:
-    image: exceptionless/elasticsearch:7.10.0
+    image: exceptionless/elasticsearch:7.12.0
     environment:
       discovery.type: single-node
       xpack.security.enabled: 'false'
@@ -17,7 +17,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:7.10.0
+    image: docker.elastic.co/kibana/kibana:7.12.0
     ports:
       - 5601:5601
 

global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
     "version": "5.0.100",
-    "rollForward": "latestMajor"
+    "rollForward": "latestMinor"
   }
 }

k8s/certificates.yaml

@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: tls-secret

k8s/cluster-issuer.yaml

@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
   name: letsencrypt-prod
@@ -14,7 +14,7 @@ spec:
           class: nginx
 
 ---
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
   name: selfsigned

k8s/ex-prod-tasks.ps1

@@ -44,17 +44,16 @@ kubectl run --namespace ex-prod ex-prod-client --rm --tty -i --restart='Never' `
 # upgrade nginx ingress to latest
 # https://github.com/kubernetes/ingress-nginx/releases
 helm repo update
-helm upgrade --reset-values --namespace nginx-ingress -f nginx-values.yaml nginx-ingress stable/nginx-ingress --dry-run
+helm upgrade --reset-values --namespace ingress-nginx -f nginx-values.yaml ingress-nginx ingress-nginx/ingress-nginx --dry-run
 
 # upgrade cert-manager
 # https://github.com/jetstack/cert-manager/releases
 helm repo update
-kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.crds.yaml
 helm upgrade cert-manager jetstack/cert-manager --namespace cert-manager --reset-values --set ingressShim.defaultIssuerName=letsencrypt-prod --set ingressShim.defaultIssuerKind=ClusterIssuer --dry-run
 
 # upgrade dashboard
 # https://github.com/kubernetes/dashboard/releases
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.3/aio/deploy/recommended.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.1.0/aio/deploy/recommended.yaml
 
 # upgrade kubecost
 helm repo update
@@ -63,11 +62,12 @@ helm upgrade kubecost kubecost/cost-analyzer --namespace kubecost --reset-values
 # upgrade goldilocks
 helm repo update
 helm upgrade goldilocks fairwinds-stable/goldilocks --namespace goldilocks --reset-values --dry-run
+helm upgrade vpa fairwinds-stable/vpa --namespace vpa -f vpa-values.yaml --reset-values --dry-run
 
 # upgrade elasticsearch operator
 # https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html
 # https://github.com/elastic/cloud-on-k8s/releases
-kubectl apply -f https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml
+kubectl apply -f https://download.elastic.co/downloads/eck/1.3.1/all-in-one.yaml
 
 # upgrade elasticsearch
 kubectl apply --namespace ex-prod -f ex-prod-elasticsearch.yaml

k8s/ex-setup.ps1

@@ -10,7 +10,7 @@ choco install azure-cli
 
 # install helm
 choco install kubernetes-helm
-helm repo add stable https://kubernetes-charts.storage.googleapis.com/
+helm repo add "stable" "https://charts.helm.sh/stable" --force-update
 helm repo add jetstack https://charts.jetstack.io
 helm repo update
 
@@ -48,7 +48,7 @@ az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER --overwr
 
 # install dashboard
 # https://github.com/kubernetes/dashboard/releases
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.3/aio/deploy/recommended.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.1.0/aio/deploy/recommended.yaml
 
 # create admin user to login to the dashboard
 kubectl apply -f admin-service-account.yaml
@@ -59,7 +59,7 @@ kubectl config set-context --current --namespace=ex-$ENV
 # setup elasticsearch operator
 # https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html
 # https://github.com/elastic/cloud-on-k8s/releases
-kubectl apply -f https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml
+kubectl apply -f https://download.elastic.co/downloads/eck/1.3.1/all-in-one.yaml
 
 # view ES operator logs
 kubectl -n elastic-system logs -f statefulset.apps/elastic-operator
@@ -98,11 +98,11 @@ curl -X PUT -H "Content-Type: application/json" -k `
 Remove-Job $ELASTIC_JOB
 
 # install nginx ingress
-helm install nginx-ingress stable/nginx-ingress --namespace nginx-ingress --values nginx-values.yaml
+helm install --namespace ingress-nginx -f nginx-values.yaml ingress-nginx ingress-nginx/ingress-nginx
 
 # wait for external ip to be assigned
-kubectl get service -l app=nginx-ingress --namespace nginx-ingress
-$IP=$(kubectl get service -l app=nginx-ingress --namespace nginx-ingress -o=jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
+kubectl get service -l app.kubernetes.io/name=ingress-nginx --namespace ingress-nginx
+$IP=$(kubectl get service -l app.kubernetes.io/name=ingress-nginx --namespace ingress-nginx -o=jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
 $PUBLICIPID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$IP')].[id]" --output tsv)
 az network public-ip update --ids $PUBLICIPID --dns-name $CLUSTER
 
@@ -117,11 +117,14 @@ helm install cert-manager jetstack/cert-manager --namespace cert-manager --set i
 # https://kubecost.com/install?ref=home
 kubectl create namespace kubecost
 helm repo add kubecost https://kubecost.github.io/cost-analyzer/
+$KUBECOST_KEY=""
 helm install kubecost kubecost/cost-analyzer --namespace kubecost --set kubecostToken=$KUBECOST_KEY
 
 # install goldilocks
 helm repo add fairwinds-stable https://charts.fairwinds.com/stable
+helm install vpa fairwinds-stable/vpa --namespace vpa --create-namespace -f vpa-values.yaml
 helm install goldilocks fairwinds-stable/goldilocks --namespace goldilocks
+kubectl label ns ex-$ENV goldilocks.fairwinds.com/enabled=true
 
 # TODO: update this file using the cluster name for the dns
 kubectl apply -f certificates.yaml

k8s/vpa-values.yaml

@@ -0,0 +1,5 @@
+recommender:
+  extraArgs:
+    prometheus-address: |
+      http://kubecost-prometheus-server.kubecost.svc.cluster.local
+    storage: prometheus

src/Exceptionless.Core/Authentication/ActiveDirectoryLoginProvider.cs

@@ -1,4 +1,5 @@
-using System.DirectoryServices;
+# pragma warning disable CA1416
+using System.DirectoryServices;
 using Exceptionless.Core.Configuration;
 
 namespace Exceptionless.Core.Authentication {
@@ -60,4 +61,5 @@ private SearchResult FindUser(string username) {
             }
         }
     }
-}
+}
+# pragma warning restore CA1416

src/Exceptionless.Core/Exceptionless.Core.csproj

@@ -23,20 +23,20 @@
   <ItemGroup>
     <PackageReference Include="AutoMapper" Version="10.1.1" />
     <PackageReference Include="AutoMapper.Collection" Version="7.0.1" />
-    <PackageReference Include="FluentValidation" Version="9.3.0" />
-    <PackageReference Include="Foundatio.Extensions.Hosting" Version="10.0.1" />
-    <PackageReference Include="Foundatio.JsonNet" Version="10.0.1" />
-    <PackageReference Include="NEST.JsonNetSerializer" Version="7.10.0" />
-    <PackageReference Include="Handlebars.Net" Version="1.11.5" />
-    <PackageReference Include="McSherry.SemanticVersioning" Version="1.4.0" />
+    <PackageReference Include="FluentValidation" Version="9.5.3" />
+    <PackageReference Include="Foundatio.Extensions.Hosting" Version="10.1.1" />
+    <PackageReference Include="Foundatio.JsonNet" Version="10.1.1" />
+    <PackageReference Include="NEST.JsonNetSerializer" Version="7.12.0" />
+    <PackageReference Include="Handlebars.Net" Version="2.0.7" />
+    <PackageReference Include="McSherry.SemanticVersioning" Version="1.4.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="5.0.5" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
-    <PackageReference Include="Stripe.net" Version="39.27.0" />
+    <PackageReference Include="Stripe.net" Version="39.44.0" />
     <PackageReference Include="System.DirectoryServices" Version="5.0.0" />
-    <PackageReference Include="UAParser" Version="3.1.44" />
+    <PackageReference Include="UAParser" Version="3.1.46" />
 
-    <PackageReference Include="Foundatio.Repositories.Elasticsearch" Version="7.10.0" Condition="'$(ReferenceFoundatioRepositoriesSource)' == '' OR '$(ReferenceFoundatioRepositoriesSource)' == 'false'" />
+    <PackageReference Include="Foundatio.Repositories.Elasticsearch" Version="7.12.1" Condition="'$(ReferenceFoundatioRepositoriesSource)' == '' OR '$(ReferenceFoundatioRepositoriesSource)' == 'false'" />
     <ProjectReference Include="..\..\..\Foundatio.Repositories\src\Foundatio.Repositories.Elasticsearch\Foundatio.Repositories.Elasticsearch.csproj" Condition="'$(ReferenceFoundatioRepositoriesSource)' == 'true'" />
   </ItemGroup>
 </Project>

src/Exceptionless.Core/Extensions/QueryNodeExtensions.cs

@@ -0,0 +1,23 @@
+using System;
+using Foundatio.Parsers.LuceneQueries.Nodes;
+
+namespace Exceptionless.Core.Extensions {
+    public static class QueryNodeExtensions {
+        public static GroupNode GetParent(this IQueryNode node, Func<GroupNode, bool> condition) {
+            if (node == null)
+                return null;
+
+            IQueryNode queryNode = node;
+            do {
+                GroupNode groupNode = queryNode as GroupNode;
+                if (groupNode != null && condition(groupNode))
+                    return groupNode;
+
+                queryNode = queryNode.Parent;
+            }
+            while (queryNode != null);
+
+            return null;
+        }
+    }
+}

src/Exceptionless.Core/Mail/Mailer.cs

@@ -17,7 +17,7 @@
 
 namespace Exceptionless.Core.Mail {
     public class Mailer : IMailer {
-        private readonly ConcurrentDictionary<string, Func<object, string>> _cachedTemplates = new ConcurrentDictionary<string, Func<object, string>>();
+        private readonly ConcurrentDictionary<string, HandlebarsTemplate<object, object>> _cachedTemplates = new ConcurrentDictionary<string, HandlebarsTemplate<object, object>>();
         private readonly IQueue<MailMessage> _queue;
         private readonly FormattingPluginManager _pluginManager;
         private readonly AppOptions _appOptions;
@@ -134,10 +134,11 @@ public Task SendOrganizationInviteAsync(User sender, Organization organization,
                 { "InviteToken", invite.Token }
             };
 
+            var body = RenderTemplate(template, data);
             return QueueMessageAsync(new MailMessage {
                 To = invite.EmailAddress,
                 Subject = subject,
-                Body = RenderTemplate(template, data)
+                Body = body
             }, template);
         }
 
@@ -261,18 +262,20 @@ public Task SendUserPasswordResetAsync(User user) {
 
         private string RenderTemplate(string name, IDictionary<string, object> data) {
             var template = GetCompiledTemplate(name);
-            return template(data);
+            var result = template(data);
+            return result?.ToString();
         }
 
-        private Func<object, string> GetCompiledTemplate(string name) {
+        private HandlebarsTemplate<object, object> GetCompiledTemplate(string name) {
             return _cachedTemplates.GetOrAdd(name, templateName => {
                 var assembly = typeof(Mailer).Assembly;
                 string resourceName = $"Exceptionless.Core.Mail.Templates.{templateName}.html";
 
                 using (var stream = assembly.GetManifestResourceStream(resourceName)) {
                     using (var reader = new StreamReader(stream)) {
                         string template = reader.ReadToEnd();
-                        return Handlebars.Compile(template);
+                        var compiledTemplateFunc = Handlebars.Compile(template);
+                        return compiledTemplateFunc;
                     }
                 }
             });

src/Exceptionless.Core/Models/Stack.cs

@@ -8,7 +8,7 @@
 using Newtonsoft.Json.Converters;
 
 namespace Exceptionless.Core.Models {
-    [DebuggerDisplay("Id: {Id}, Type: {Type}, Title: {Title}, TotalOccurrences: {TotalOccurrences}")]
+    [DebuggerDisplay("Id={Id} Type={Type} Status={Status} IsDeleted={IsDeleted} Title={Title} TotalOccurrences={TotalOccurrences}")]
     public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISupportSoftDeletes {
         public Stack() {
             Tags = new TagSet();

src/Exceptionless.Core/Models/StackSummaryModel.cs

@@ -1,6 +1,8 @@
 using System;
+using System.Diagnostics;
 
 namespace Exceptionless.Core.Models {
+    [DebuggerDisplay("Id: {Id}, Status: {Status}, Title: {Title}, First: {FirstOccurrence}, Last: {LastOccurrence}")]
     public class StackSummaryModel : SummaryData {
         public string Id { get; set; }
         public string Title { get; set; }

src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Exceptionless.Core.Configuration;
-using Exceptionless.Core.Extensions;
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Models.Data;
 using Exceptionless.Core.Repositories.Queries;

src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs

@@ -103,11 +103,11 @@ public AppFilterQueryBuilder(AppOptions options) {
             }
 
             var index = ctx.Options.GetElasticIndex();
-            bool shouldApplyRetentionFilter = ShouldApplyRetentionFilter(index);
+            bool shouldApplyRetentionFilter = ShouldApplyRetentionFilter(index, ctx);
             string field = shouldApplyRetentionFilter ? GetDateField(index) : null;
 
             if (sfq.Stack != null) {
-                var stackIdFieldName = typeof(T) == typeof(Stack) ? "id" : _stackIdFieldName;
+                string stackIdFieldName = typeof(T) == typeof(Stack) ? "id" : _stackIdFieldName;
                 var organization = allowedOrganizations.SingleOrDefault(o => o.Id == sfq.Stack.OrganizationId);
                 if (organization != null) {
                     if (shouldApplyRetentionFilter)
@@ -161,13 +161,13 @@ public AppFilterQueryBuilder(AppOptions options) {
             return Query<T>.DateRange(r => r.Field(field).GreaterThanOrEquals($"now/d-{(int)retentionDays}d").LessThanOrEquals("now/d+1d"));
         }
 
-        private bool ShouldApplyRetentionFilter(IIndex index) {
+        private bool ShouldApplyRetentionFilter<T>(IIndex index, QueryBuilderContext<T> ctx) where T : class, new() {
             if (index == null)
                 throw new ArgumentNullException(nameof(index));
-            
+
             var indexType = index.GetType();
             if (indexType == typeof(StackIndex))
-                return true;
+                return !ctx.Source.IsEventStackFilterInverted();
 
             if (indexType == typeof(EventIndex))
                 return true;