From f757093be457762dfd9da25f8ff135833cc0f42b Mon Sep 17 00:00:00 2001 From: Karol Zadora-Przylecki Date: Fri, 6 Sep 2019 19:49:50 -0700 Subject: [PATCH] Support for Elasticsearch 7 (#331) --- README.md | 12 +++-- .../ElasticSearchOutput.cs | 53 +++++++------------ ...ics.EventFlow.Outputs.ElasticSearch.csproj | 13 +++-- ...osoft.Diagnostics.EventFlow.Signing.csproj | 6 +-- .../DiagnosticsPipelineFactoryTests.cs | 13 ++--- ...ft.Diagnostics.EventFlow.Core.Tests.csproj | 13 +++-- ....Diagnostics.EventFlow.Inputs.Tests.csproj | 8 +-- ...Diagnostics.EventFlow.Outputs.Tests.csproj | 6 +-- 8 files changed, 50 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index ca29b0b4..eea7fc9a 100644 --- a/README.md +++ b/README.md @@ -792,8 +792,6 @@ All other events will be reported as Application Insights *traces* (telemetry of #### Elasticsearch *Nuget Package*: [**Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch**](https://www.nuget.org/packages/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/) -**Note: Nuget package version 1.x supports Elasticsearch version 2.x. Nuget package version 2.x supports Elasticsearch version 6.x** - This output writes data to the [Elasticsearch](https://www.elastic.co/products/elasticsearch). Here is an example showing all possible settings: ```json { @@ -803,7 +801,6 @@ This output writes data to the [Elasticsearch](https://www.elastic.co/products/e "connectionPoolType": "Sniffing", "basicAuthenticationUserName": "esUser1", "basicAuthenticationUserPassword": "", - "eventDocumentTypeName": "diagData", "numberOfShards": 1, "numberOfReplicas": 1, "refreshInterval": "15s", @@ -818,7 +815,7 @@ This output writes data to the [Elasticsearch](https://www.elastic.co/products/e | `connectionPoolType` | "Static", "Sniffing", or "Sticky" | No | Specifies the Connection Pool that takes care of registering what nodes there are in the cluster. | | `basicAuthenticationUserName` | string | No | Specifies the user name used to authenticate with Elasticsearch. To protect the cluster, authentication is often setup on the cluster. | | `basicAuthenticationUserPassword` | string | No | Specifies the password used to authenticate with Elasticsearch. This field should be used only if basicAuthenticationUserName is specified. | -| `eventDocumentTypeName` | string | Yes | Specifies the document type to be applied when data is written. Elasticsearch allows documents to be typed, so they can be distinguished from other types. This type name is user-defined. | +| `eventDocumentTypeName` | string | Yes (ver < 2.7.0)
N/A (ver >= 2.7.0) | Specifies the document type to be applied when data is written. Elasticsearch allows documents to be typed, so they can be distinguished from other types. This type name is user-defined.

Starting with Elasticsearch 7.x the [mapping types have been removed](https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html). Consequently this configuration setting has been removed from Elasticsearch output version 2.7.0 and newer. | | `numberOfShards` | int | No | Specifies how many shards to create the index with. If not specified, it defaults to 1.| | `numberOfReplicas` | int | No | Specifies how many replicas the index is created with. If not specified, it defaults to 5.| | `refreshInterval` | string | No | Specifies what refresh interval the index is created with. If not specified, it defaults to 15s.| @@ -845,6 +842,13 @@ Fields injected byt the `request` metadata are: | `IsSuccess` | Success indicator, read from the event property specified by `isSuccessProperty` (if available). | | `ResponseCode` | Response code for the request, read from the event property specified by `responseCodeProperty` (if available). | +*Elasticsearch version support* +| Elasticsearch output package version | Supported Elasticsearch server version | +| :---- | :---- | +| 1.x | 2.x | +| 2.6.x | 6.x | +| 2.7.x | 7.x | + #### Azure Monitor Logs *Nuget package*: [**Microsoft.Diagnostics.EventFlow.Outputs.AzureMonitorLogs**](https://www.nuget.org/packages/Microsoft.Diagnostics.EventFlow.Outputs.AzureMonitorLogs/) diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/ElasticSearchOutput.cs b/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/ElasticSearchOutput.cs index ca50b407..42b55728 100644 --- a/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/ElasticSearchOutput.cs +++ b/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/ElasticSearchOutput.cs @@ -98,7 +98,7 @@ public async Task SendEventsAsync(IReadOnlyCollection events, long tr // Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance // between different SendEventsAsync callbacks. // Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3 - IBulkResponse response = await this.connectionData.Client.BulkAsync(request).ConfigureAwait(false); + BulkResponse response = await this.connectionData.Client.BulkAsync(request).ConfigureAwait(false); if (!response.IsValid) { this.ReportEsRequestError(response, "Bulk upload"); @@ -130,7 +130,7 @@ private IEnumerable GetCreateOperationsForEvent(EventData eventD { foreach (var metricMetadata in metadataSet) { - operation = CreateMetricOperation(eventData, metricMetadata, currentIndexName, documentTypeName); + operation = CreateMetricOperation(eventData, metricMetadata, currentIndexName); if (operation != null) { reportedAsSpecialEvent = true; @@ -143,7 +143,7 @@ private IEnumerable GetCreateOperationsForEvent(EventData eventD { foreach (var requestMetadata in metadataSet) { - operation = CreateRequestOperation(eventData, requestMetadata, currentIndexName, documentTypeName); + operation = CreateRequestOperation(eventData, requestMetadata, currentIndexName); if (operation != null) { reportedAsSpecialEvent = true; @@ -156,7 +156,7 @@ private IEnumerable GetCreateOperationsForEvent(EventData eventD { foreach (var dependencyMetadata in metadataSet) { - operation = CreateDependencyOperation(eventData, dependencyMetadata, currentIndexName, documentTypeName); + operation = CreateDependencyOperation(eventData, dependencyMetadata, currentIndexName); if (operation != null) { reportedAsSpecialEvent = true; @@ -169,7 +169,7 @@ private IEnumerable GetCreateOperationsForEvent(EventData eventD { foreach (var dependencyMetadata in metadataSet) { - operation = CreateDependencyOperation(eventData, dependencyMetadata, currentIndexName, documentTypeName); + operation = CreateDependencyOperation(eventData, dependencyMetadata, currentIndexName); if (operation != null) { reportedAsSpecialEvent = true; @@ -182,7 +182,7 @@ private IEnumerable GetCreateOperationsForEvent(EventData eventD { foreach (var exceptionMetadata in metadataSet) { - operation = CreateExceptionOperation(eventData, exceptionMetadata, currentIndexName, documentTypeName); + operation = CreateExceptionOperation(eventData, exceptionMetadata, currentIndexName); if (operation != null) { reportedAsSpecialEvent = true; @@ -193,16 +193,12 @@ private IEnumerable GetCreateOperationsForEvent(EventData eventD if (!reportedAsSpecialEvent) { - operation = CreateOperation(eventData, currentIndexName, documentTypeName); + operation = CreateOperation(eventData, currentIndexName); yield return operation; } } - private BulkIndexOperation CreateMetricOperation( - EventData eventData, - EventMetadata metricMetadata, - string currentIndexName, - string documentTypeName) + private BulkIndexOperation CreateMetricOperation(EventData eventData, EventMetadata metricMetadata, string currentIndexName) { var result = MetricData.TryGetData(eventData, metricMetadata, out MetricData metricData); if (result.Status != DataRetrievalStatus.Success) @@ -214,15 +210,11 @@ private BulkIndexOperation CreateMetricOperation( var metricEventData = eventData.DeepClone(); metricEventData.Payload[nameof(MetricData.MetricName)] = metricData.MetricName; metricEventData.Payload[nameof(MetricData.Value)] = metricData.Value; - var operation = CreateOperation(metricEventData, currentIndexName, documentTypeName); + var operation = CreateOperation(metricEventData, currentIndexName); return operation; } - private BulkIndexOperation CreateRequestOperation( - EventData eventData, - EventMetadata requestMetadata, - string currentIndexName, - string documentTypeName) + private BulkIndexOperation CreateRequestOperation(EventData eventData, EventMetadata requestMetadata, string currentIndexName) { var result = RequestData.TryGetData(eventData, requestMetadata, out RequestData requestData); if (result.Status != DataRetrievalStatus.Success) @@ -245,15 +237,11 @@ private BulkIndexOperation CreateRequestOperation( { requestEventData.Payload[nameof(RequestData.ResponseCode)] = requestData.ResponseCode; } - var operation = CreateOperation(requestEventData, currentIndexName, documentTypeName); + var operation = CreateOperation(requestEventData, currentIndexName); return operation; } - private BulkIndexOperation CreateDependencyOperation( - EventData eventData, - EventMetadata dependencyMetadata, - string currentIndexName, - string documentTypeName) + private BulkIndexOperation CreateDependencyOperation(EventData eventData, EventMetadata dependencyMetadata, string currentIndexName) { var result = DependencyData.TryGetData(eventData, dependencyMetadata, out DependencyData dependencyData); if (result.Status != DataRetrievalStatus.Success) @@ -283,15 +271,11 @@ private BulkIndexOperation CreateDependencyOperation( { dependencyEventData.Payload[nameof(DependencyData.DependencyType)] = dependencyData.DependencyType; } - var operation = CreateOperation(dependencyEventData, currentIndexName, documentTypeName); + var operation = CreateOperation(dependencyEventData, currentIndexName); return operation; } - private BulkIndexOperation CreateExceptionOperation( - EventData eventData, - EventMetadata exceptionMetadata, - string currentIndexName, - string documentTypeName) + private BulkIndexOperation CreateExceptionOperation(EventData eventData, EventMetadata exceptionMetadata, string currentIndexName) { var result = ExceptionData.TryGetData(eventData, exceptionMetadata, out ExceptionData exceptionData); if (result.Status != DataRetrievalStatus.Success) @@ -302,15 +286,14 @@ private BulkIndexOperation CreateExceptionOperation( var exceptionEventData = eventData.DeepClone(); exceptionEventData.Payload[nameof(ExceptionData.Exception)] = exceptionData.Exception.ToString(); - var operation = CreateOperation(exceptionEventData, currentIndexName, documentTypeName); + var operation = CreateOperation(exceptionEventData, currentIndexName); return operation; } - private static BulkIndexOperation CreateOperation(EventData eventData, string currentIndexName, string documentTypeName) + private static BulkIndexOperation CreateOperation(EventData eventData, string currentIndexName) { BulkIndexOperation operation = new BulkIndexOperation(eventData); operation.Index = currentIndexName; - operation.Type = documentTypeName; return operation; } @@ -371,7 +354,7 @@ private void Initialize(ElasticSearchOutputConfiguration esOutputConfiguration) private async Task EnsureIndexExists(string indexName, ElasticClient esClient) { - IExistsResponse existsResult = await esClient.IndexExistsAsync(indexName).ConfigureAwait(false); + ExistsResponse existsResult = await esClient.Indices.ExistsAsync(indexName).ConfigureAwait(false); if (!existsResult.IsValid) { this.ReportEsRequestError(existsResult, "Index exists check"); @@ -393,7 +376,7 @@ private async Task EnsureIndexExists(string indexName, ElasticClient esClient) indexSettings.Settings.Add("default_pipeline", this.connectionData.Configuration.DefaultPipeline); } - ICreateIndexResponse createIndexResult = await esClient.CreateIndexAsync(indexName, c => c.InitializeUsing(indexSettings)).ConfigureAwait(false); + CreateIndexResponse createIndexResult = await esClient.Indices.CreateAsync(indexName, c => c.InitializeUsing(indexSettings)).ConfigureAwait(false); if (!createIndexResult.IsValid) { diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch.csproj b/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch.csproj index ac58057a..608d4828 100644 --- a/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch.csproj +++ b/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch.csproj @@ -3,9 +3,9 @@ Provides an output implementation that sends diagnostics data to Elasticsearch. © Microsoft Corporation. All rights reserved. - 2.6.1 + 2.7.0 Microsoft - netstandard1.6;net451;netstandard2.0 + net471;netstandard2.0 Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch true Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch @@ -13,8 +13,7 @@ https://github.com/Azure/diagnostics-eventflow MIT true - 1.6.1 - 1.6.1 + 2.0.0 2.0.0 false false @@ -27,11 +26,11 @@ - - + + - + diff --git a/src/Microsoft.Diagnostics.EventFlow.Signing/Microsoft.Diagnostics.EventFlow.Signing.csproj b/src/Microsoft.Diagnostics.EventFlow.Signing/Microsoft.Diagnostics.EventFlow.Signing.csproj index 3fd42a4b..a7a2ca43 100644 --- a/src/Microsoft.Diagnostics.EventFlow.Signing/Microsoft.Diagnostics.EventFlow.Signing.csproj +++ b/src/Microsoft.Diagnostics.EventFlow.Signing/Microsoft.Diagnostics.EventFlow.Signing.csproj @@ -77,7 +77,6 @@ ..\Microsoft.Diagnostics.EventFlow.Inputs.Log4net\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.Inputs.Log4net.dll; ..\Microsoft.Diagnostics.EventFlow.Inputs.NLog\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.Inputs.NLog.dll; ..\Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights.dll; - ..\Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch.dll; ..\Microsoft.Diagnostics.EventFlow.Outputs.EventHub\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.Outputs.EventHub.dll; ..\Microsoft.Diagnostics.EventFlow.Outputs.StdOutput\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.Outputs.StdOutput.dll; ..\Microsoft.Diagnostics.EventFlow.Outputs.Oms\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.Outputs.Oms.dll; @@ -86,6 +85,7 @@ ..\Microsoft.Diagnostics.EventFlow.ServiceFabric\bin\$(Configuration)\net451\Microsoft.Diagnostics.EventFlow.ServiceFabric.dll;" /> @@ -152,7 +151,6 @@ - @@ -162,6 +160,7 @@ + @@ -177,7 +176,6 @@ - diff --git a/test/Microsoft.Diagnostics.EventFlow.Core.Tests/DiagnosticsPipelineFactoryTests.cs b/test/Microsoft.Diagnostics.EventFlow.Core.Tests/DiagnosticsPipelineFactoryTests.cs index 28d24611..4cfa1f48 100644 --- a/test/Microsoft.Diagnostics.EventFlow.Core.Tests/DiagnosticsPipelineFactoryTests.cs +++ b/test/Microsoft.Diagnostics.EventFlow.Core.Tests/DiagnosticsPipelineFactoryTests.cs @@ -314,7 +314,7 @@ public void CanOverrideDefaultPipelineItems() [Fact] public void CanCreateAllStandardPipelineItems() { -#if NET46 +#if NET461 string pipelineConfiguration = @" { ""inputs"": [ @@ -361,11 +361,6 @@ public void CanCreateAllStandardPipelineItems() { ""type"": ""StdOutput"", }, - { - ""type"": ""ElasticSearch"", - ""serviceUri"": ""https://myElasticSearchCluster:9200"", - ""eventDocumentTypeName"": ""diagData"" - }, { ""type"": ""OmsOutput"", ""workspaceId"": ""00000000-0000-0000-0000-000000000000"", @@ -469,7 +464,7 @@ public void CanCreateAllStandardPipelineItems() i => Assert.IsType(i), i => Assert.IsType(i), i => Assert.IsType(i) -#if NET46 +#if NET461 , i => Assert.IsType(i) , i => Assert.IsType(i) #endif @@ -477,11 +472,13 @@ public void CanCreateAllStandardPipelineItems() Assert.Collection(pipeline.Sinks, s => Assert.IsType(s.Output), +#if (!NET461) s => Assert.IsType(s.Output), +#endif s => Assert.IsType(s.Output), s => Assert.IsType(s.Output), // Azure Monitor Logs output can be created using the old and the new name s => Assert.IsType(s.Output) -#if NET46 +#if NET461 , s => Assert.IsType(s.Output) , s => Assert.IsType(s.Output) #endif diff --git a/test/Microsoft.Diagnostics.EventFlow.Core.Tests/Microsoft.Diagnostics.EventFlow.Core.Tests.csproj b/test/Microsoft.Diagnostics.EventFlow.Core.Tests/Microsoft.Diagnostics.EventFlow.Core.Tests.csproj index fe7adf85..75c7b644 100644 --- a/test/Microsoft.Diagnostics.EventFlow.Core.Tests/Microsoft.Diagnostics.EventFlow.Core.Tests.csproj +++ b/test/Microsoft.Diagnostics.EventFlow.Core.Tests/Microsoft.Diagnostics.EventFlow.Core.Tests.csproj @@ -19,13 +19,12 @@ - - + - + @@ -36,6 +35,10 @@ + + + + @@ -43,8 +46,8 @@ - - + + diff --git a/test/Microsoft.Diagnostics.EventFlow.Inputs.Tests/Microsoft.Diagnostics.EventFlow.Inputs.Tests.csproj b/test/Microsoft.Diagnostics.EventFlow.Inputs.Tests/Microsoft.Diagnostics.EventFlow.Inputs.Tests.csproj index c9747e65..efc1ce67 100644 --- a/test/Microsoft.Diagnostics.EventFlow.Inputs.Tests/Microsoft.Diagnostics.EventFlow.Inputs.Tests.csproj +++ b/test/Microsoft.Diagnostics.EventFlow.Inputs.Tests/Microsoft.Diagnostics.EventFlow.Inputs.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp2.0;net462;net471 + netcoreapp2.0;net471 portable Microsoft.Diagnostics.EventFlow.Inputs.Tests true @@ -27,12 +27,6 @@ - - - - - - diff --git a/test/Microsoft.Diagnostics.EventFlow.Outputs.Tests/Microsoft.Diagnostics.EventFlow.Outputs.Tests.csproj b/test/Microsoft.Diagnostics.EventFlow.Outputs.Tests/Microsoft.Diagnostics.EventFlow.Outputs.Tests.csproj index 83daa77c..5e76fd55 100644 --- a/test/Microsoft.Diagnostics.EventFlow.Outputs.Tests/Microsoft.Diagnostics.EventFlow.Outputs.Tests.csproj +++ b/test/Microsoft.Diagnostics.EventFlow.Outputs.Tests/Microsoft.Diagnostics.EventFlow.Outputs.Tests.csproj @@ -1,12 +1,10 @@  - net46;netcoreapp2.0 + net471;netcoreapp2.0 Microsoft.Diagnostics.EventFlow.Outputs.Tests true true - $(PackageTargetFallback);dotnet5.4;portable-net451+win8 - 1.1.0 $(NoWarn);NU1603 true true @@ -22,7 +20,7 @@ - +