Skip to content

Commit

Permalink
Add expiration for stored request durations
Browse files Browse the repository at this point in the history
This adds configurable expiration time for request durations. This is to prevent memory leak in long-running applications where otherwise the queue would only keep growing indefinitely.
  • Loading branch information
RReverser committed Dec 4, 2024
1 parent d2ee667 commit 4062bbf
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 4 deletions.
1 change: 1 addition & 0 deletions examples~/quickstart/client/client.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
<ImplicitUsings>disable</ImplicitUsings>
<Nullable>enable</Nullable>

Expand Down
39 changes: 36 additions & 3 deletions src/Stats.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ namespace SpacetimeDB
{
public class NetworkRequestTracker
{
private readonly ConcurrentQueue<(DateTime End, TimeSpan Duration, string Metadata)> _requestDurations = new();
private readonly ConcurrentQueue<(DateTime End, (TimeSpan Duration, string Metadata) Request)> _requestDurations = new();

private uint _nextRequestId;
private readonly Dictionary<uint, (DateTime Start, string Metadata)> _requests = new();

// Limit the number of request durations we store to prevent memory leaks.
public int KeepLastSeconds = 5 * 60;

internal uint StartTrackingRequest(string metadata = "")
{
// Record the start time of the request
Expand Down Expand Up @@ -40,9 +43,25 @@ internal bool FinishTrackingRequest(uint requestId)
return true;
}

private IEnumerable<(TimeSpan Duration, string Metadata)> GetRequestDurations(int lastSeconds)
{
var cutoff = DateTime.UtcNow.AddSeconds(-lastSeconds);
return _requestDurations.SkipWhile(x => x.End < cutoff).Select(x => x.Request);
}

internal void InsertRequest(TimeSpan duration, string metadata)
{
_requestDurations.Enqueue((DateTime.UtcNow, duration, metadata));
lock (_requestDurations)
{
// Remove expired entries, we need to do this atomically.
var cutoff = DateTime.UtcNow.AddSeconds(-KeepLastSeconds);
var removeCount = _requestDurations.TakeWhile(x => x.End < cutoff).Count();
for (var i = 0; i < removeCount; i++)
{
_requestDurations.TryDequeue(out _);
}
_requestDurations.Enqueue((DateTime.UtcNow, (duration, metadata)));
}
}

internal void InsertRequest(DateTime start, string metadata)
Expand All @@ -52,8 +71,13 @@ internal void InsertRequest(DateTime start, string metadata)

public ((TimeSpan Duration, string Metadata) Min, (TimeSpan Duration, string Metadata) Max)? GetMinMaxTimes(int lastSeconds)
{
if (lastSeconds > KeepLastSeconds)
{
throw new ArgumentException($"lastSeconds must be less than or equal to KeepLastSeconds = {KeepLastSeconds}", nameof(lastSeconds));
}

var cutoff = DateTime.UtcNow.AddSeconds(-lastSeconds);
var requestDurations = _requestDurations.Where(x => x.End >= cutoff).Select(x => (x.Duration, x.Metadata));
var requestDurations = _requestDurations.SkipWhile(x => x.End < cutoff).Select(x => x.Request);

if (!requestDurations.Any())
{
Expand All @@ -74,5 +98,14 @@ public class Stats
public readonly NetworkRequestTracker SubscriptionRequestTracker = new();
public readonly NetworkRequestTracker AllReducersTracker = new();
public readonly NetworkRequestTracker ParseMessageTracker = new();

public void KeepLastSeconds(int seconds)
{
ReducerRequestTracker.KeepLastSeconds = seconds;
OneOffRequestTracker.KeepLastSeconds = seconds;
SubscriptionRequestTracker.KeepLastSeconds = seconds;
AllReducersTracker.KeepLastSeconds = seconds;
ParseMessageTracker.KeepLastSeconds = seconds;
}
}
}
2 changes: 1 addition & 1 deletion tests~/VerifyInit.cs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public override void Write(VerifyJsonWriter writer, NetworkRequestTracker value)
}

if (
value.GetMinMaxTimes(int.MaxValue) is
value.GetMinMaxTimes(60) is
{ Min.Metadata: var Min, Max.Metadata: var Max }
)
{
Expand Down

0 comments on commit 4062bbf

Please sign in to comment.