Skip to content

Commit

Permalink
Introduce OpenTelemetry (OTEL) instrumentation (#3163)
Browse files Browse the repository at this point in the history 
Resolves #3005

This PR instruments The Combine using OpenTelemetry for observability via the following changes:

* Adds OpenTelemetry instrumentation for tracing.
* Creates a LocationProvider that extracts the IP address from the http context and returns the IP’s location. This location information is added as tags to every trace.
* Adds the session ID as a tag to every trace to group events occurring during the same session.
* Starts new activities with custom trace tags in word-related functions to demonstrate more granular tracing.
* Installs [OpenTelemetry Collector Helm Chart](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector) in Kubernetes cluster and adds custom configuration to send traces to Honeycomb and prepare foundation for additional data handling.
* Sets the OTEL_SERVICE_NAME during Kubernetes setup to select Honeycomb dataset according to the host on which The Combine is installed.

Additional changes in this PR:

* Fixes Kubernetes setup to properly interpret branch names containing '/' characters.

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/sillsdev/TheCombine/3163)
<!-- Reviewable:end -->
---------

Co-authored-by: Jim Grady <[email protected]>
Co-authored-by: Danny Rorabaugh <[email protected]>
  • Loading branch information
@andracc @jmgrady @imnasnainaec
3 people authored Oct 23, 2024
1 parent 4078438 commit ceaf711
Show file tree
Hide file tree
Showing 24 changed files with 928 additions and 12 deletions.
1 change: 1 addition & 0 deletions Backend.Tests/Backend.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="Moq.Contrib.HttpClient" Version="1.4.0" />
<PackageReference Include="NUnit" Version="4.2.2" />
<PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
<PackageReference Include="coverlet.collector" Version="6.0.2"/>
Expand Down
20 changes: 20 additions & 0 deletions Backend.Tests/Mocks/LocationProviderMock.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
using System.Threading.Tasks;
using BackendFramework.Interfaces;
using BackendFramework.Otel;

namespace Backend.Tests.Mocks
{
sealed internal class LocationProviderMock : ILocationProvider
{
public Task<LocationApi?> GetLocation()
{
LocationApi location = new LocationApi
{
Country = "test country",
RegionName = "test region",
City = "city"
};
return Task.FromResult<LocationApi?>(location);
}
}
}
116 changes: 116 additions & 0 deletions Backend.Tests/Otel/LocationProviderTests.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
using System;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using BackendFramework.Otel;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using Moq.Protected;
using NUnit.Framework;

namespace Backend.Tests.Otel
{
public class LocationProviderTests
{
private readonly IPAddress TestIpAddress = new(new byte[] { 100, 0, 0, 0 });
private IHttpContextAccessor _contextAccessor = null!;
private IMemoryCache _memoryCache = null!;
private Mock<HttpMessageHandler> _handlerMock = null!;
private Mock<IHttpClientFactory> _httpClientFactory = null!;
private LocationProvider _locationProvider = null!;

[SetUp]
public void Setup()
{
// Set up HttpContextAccessor with mocked IP
_contextAccessor = new HttpContextAccessor();
var httpContext = new DefaultHttpContext()
{
Connection =
{
RemoteIpAddress = TestIpAddress
}
};
_contextAccessor.HttpContext = httpContext;

// Set up MemoryCache
var services = new ServiceCollection();
services.AddMemoryCache();
var serviceProvider = services.BuildServiceProvider();
_memoryCache = serviceProvider.GetService<IMemoryCache>()!;

var result = new HttpResponseMessage()
{
StatusCode = HttpStatusCode.OK,
Content = new StringContent("{}")
};

// Set up HttpClientFactory mock using httpClient with mocked HttpMessageHandler
_handlerMock = new Mock<HttpMessageHandler>();
_handlerMock.Protected()
.Setup<Task<HttpResponseMessage>>(
"SendAsync",
ItExpr.IsAny<HttpRequestMessage>(),
ItExpr.IsAny<CancellationToken>()
)
.ReturnsAsync(result)
.Verifiable();

var httpClient = new HttpClient(_handlerMock.Object);
_httpClientFactory = new Mock<IHttpClientFactory>();
_httpClientFactory
.Setup(x => x.CreateClient(It.IsAny<string>()))
.Returns(httpClient);

_locationProvider = new LocationProvider(_contextAccessor, _memoryCache, _httpClientFactory.Object);
}

public static void Verify(Mock<HttpMessageHandler> mock, Func<HttpRequestMessage, bool> match)
{
mock.Protected()
.Verify(
"SendAsync",
Times.Exactly(1),
ItExpr.Is<HttpRequestMessage>(req => match(req)),
ItExpr.IsAny<CancellationToken>()
);
}

[Test]
public async Task GetLocationHttpClientUsesIp()
{
// Act
await _locationProvider.GetLocationFromIp(TestIpAddress.ToString());

// Assert
Verify(_handlerMock, r => r.RequestUri!.AbsoluteUri.Contains(TestIpAddress.ToString()));
Verify(_handlerMock, r => !r.RequestUri!.AbsoluteUri.Contains("123.1.2.3"));
}

[Test]
public async Task GetLocationUsesHttpContextIp()
{
// Act
await _locationProvider.GetLocation();

// Assert
Verify(_handlerMock, r => r.RequestUri!.AbsoluteUri.Contains(TestIpAddress.ToString()));
Verify(_handlerMock, r => !r.RequestUri!.AbsoluteUri.Contains("123.1.2.3"));
}

[Test]
public async Task GetLocationUsesCache()
{
// Act
// call getLocation twice and verify async method is called only once
await _locationProvider.GetLocation();
await _locationProvider.GetLocation();

// Assert
Verify(_handlerMock, r => r.RequestUri!.AbsoluteUri.Contains(TestIpAddress.ToString()));
}
}
}
99 changes: 99 additions & 0 deletions Backend.Tests/Otel/OtelKernelTests.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Backend.Tests.Mocks;
using BackendFramework.Otel;
using Microsoft.AspNetCore.Http;
using NUnit.Framework;
using static BackendFramework.Otel.OtelKernel;

namespace Backend.Tests.Otel
{
public class OtelKernelTests : IDisposable
{
private const string FrontendSessionIdKey = "sessionId";
private const string OtelSessionIdKey = "sessionId";
private const string OtelSessionBaggageKey = "sessionBaggage";
private LocationEnricher _locationEnricher = null!;

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}

protected virtual void Dispose(bool disposing)
{
if (disposing)
{
_locationEnricher?.Dispose();
}
}

[Test]
public void BuildersSetSessionBaggageFromHeader()
{
// Arrange
var httpContext = new DefaultHttpContext();
httpContext.Request.Headers[FrontendSessionIdKey] = "123";
var activity = new Activity("testActivity").Start();

// Act
TrackSession(activity, httpContext.Request);

// Assert
Assert.That(activity.Baggage.Any(_ => _.Key == OtelSessionBaggageKey));
}

[Test]
public void OnEndSetsSessionTagFromBaggage()
{
// Arrange
var activity = new Activity("testActivity").Start();
activity.SetBaggage(OtelSessionBaggageKey, "test session id");

// Act
_locationEnricher.OnEnd(activity);

// Assert
Assert.That(activity.Tags.Any(_ => _.Key == OtelSessionIdKey));
}


[Test]
public void OnEndSetsLocationTags()
{
// Arrange
_locationEnricher = new LocationEnricher(new LocationProviderMock());
var activity = new Activity("testActivity").Start();

// Act
_locationEnricher.OnEnd(activity);

// Assert
var testLocation = new Dictionary<string, string>
{
{"country", "test country"},
{"regionName", "test region"},
{"city", "city"}
};
Assert.That(activity.Tags, Is.SupersetOf(testLocation));
}

public void OnEndRedactsIp()
{
// Arrange
_locationEnricher = new LocationEnricher(new LocationProviderMock());
var activity = new Activity("testActivity").Start();
activity.SetTag("url.full", $"{LocationProvider.locationGetterUri}100.0.0.0");

// Act
_locationEnricher.OnEnd(activity);

// Assert
Assert.That(activity.Tags.Any(_ => _.Key == "url.full" && _.Value == ""));
Assert.That(activity.Tags.Any(_ => _.Key == "url.redacted.ip" && _.Value == LocationProvider.locationGetterUri));
}
}
}
39 changes: 39 additions & 0 deletions Backend.Tests/Otel/OtelServiceTests.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
using System.Diagnostics;
using BackendFramework.Otel;
using Microsoft.Extensions.DependencyInjection;
using NUnit.Framework;

namespace Backend.Tests.Otel
{
public class OtelServiceTests
{
[Test]
public static void TestStartActivityWithTag()
{
// Arrange
var services = new ServiceCollection();
services.AddOpenTelemetryInstrumentation();
AddActivityListener();

// Act
var activity = OtelService.StartActivityWithTag("test key", "test val");
var tag = activity?.GetTagItem("test key");
var wrongTag = activity?.GetTagItem("wrong key");

// Assert
Assert.That(activity, Is.Not.Null);
Assert.That(tag, Is.Not.Null);
Assert.That(wrongTag, Is.Null);
}

private static void AddActivityListener()
{
var activityListener = new ActivityListener
{
ShouldListenTo = _ => true,
Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllData
};
ActivitySource.AddActivityListener(activityListener);
}
}
}
10 changes: 8 additions & 2 deletions Backend/BackendFramework.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<LangVersion>10.0</LangVersion>
<LangVersion>12.0</LangVersion>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<AnalysisMode>Recommended</AnalysisMode>
Expand All @@ -10,7 +10,13 @@
<NoWarn>$(NoWarn);CA1305;CA1848;CS1591</NoWarn>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="RelaxNG" Version="3.2.3" >
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
<PackageReference Include="RelaxNG" Version="3.2.3">
<NoWarn>NU1701</NoWarn>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.3" />
Expand Down
Loading

0 comments on commit ceaf711

Please sign in to comment.