Skip to content

Commit

Permalink
Add RetryAfter to BrokenCircuitException.
Browse files Browse the repository at this point in the history
  • Loading branch information
DL444 committed Oct 2, 2024
1 parent 74ac188 commit 8ede4af
Show file tree
Hide file tree
Showing 18 changed files with 535 additions and 153 deletions.
34 changes: 34 additions & 0 deletions src/Polly.Core/CircuitBreaker/BrokenCircuitException.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ public BrokenCircuitException()
{
}

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException"/> class.
/// </summary>
/// <param name="retryAfter">The time during which the circuit will be broken.</param>
public BrokenCircuitException(TimeSpan retryAfter)
: base($"The circuit is now open and is not allowing calls. It can be retried after '{retryAfter}'.")
=> RetryAfter = retryAfter;

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException"/> class.
/// </summary>
Expand All @@ -31,6 +39,14 @@ public BrokenCircuitException(string message)
{
}

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException"/> class.
/// </summary>
/// <param name="message">The message that describes the error.</param>
/// <param name="retryAfter">The time during which the circuit will be broken.</param>
public BrokenCircuitException(string message, TimeSpan retryAfter)
: base(message) => RetryAfter = retryAfter;

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException"/> class.
/// </summary>
Expand All @@ -41,6 +57,15 @@ public BrokenCircuitException(string message, Exception inner)
{
}

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException"/> class.
/// </summary>
/// <param name="message">The message that describes the error.</param>
/// <param name="inner">The inner exception.</param>
/// <param name="retryAfter">The time during which the circuit will be broken.</param>
public BrokenCircuitException(string message, Exception inner, TimeSpan retryAfter)
: base(message, inner) => RetryAfter = retryAfter;

#pragma warning disable RS0016 // Add public types and members to the declared API
#if !NETCOREAPP
/// <summary>
Expand All @@ -54,4 +79,13 @@ protected BrokenCircuitException(SerializationInfo info, StreamingContext contex
}
#endif
#pragma warning restore RS0016 // Add public types and members to the declared API

/// <summary>
/// Gets the amount of time before circuit can become closed.
/// </summary>
/// <remarks>
/// This is set when the exception object was constructed and might be inaccurate if consumed at a later time.
/// Can be <see langword="null"/> if not provided or when the circuit was manually isolated.
/// </remarks>
public TimeSpan? RetryAfter { get; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -307,8 +307,8 @@ private void SetLastHandledOutcome_NeedsLock(Outcome<T> outcome)

private BrokenCircuitException CreateBrokenCircuitException() => _breakingException switch
{
Exception exception => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, exception),
_ => new BrokenCircuitException(BrokenCircuitException.DefaultMessage)
Exception exception => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, exception, _blockedUntil - _timeProvider.GetUtcNow()),
_ => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, _blockedUntil - _timeProvider.GetUtcNow())
};

private void OpenCircuit_NeedsLock(Outcome<T> outcome, bool manual, ResilienceContext context, out Task? scheduledTask)
Expand Down
4 changes: 4 additions & 0 deletions src/Polly.Core/PublicAPI.Unshipped.txt
Original file line number Diff line number Diff line change
@@ -1 +1,5 @@
#nullable enable
Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! message, System.Exception! inner, System.TimeSpan retryAfter) -> void
Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! message, System.TimeSpan retryAfter) -> void
Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(System.TimeSpan retryAfter) -> void
Polly.CircuitBreaker.BrokenCircuitException.RetryAfter.get -> System.TimeSpan?
27 changes: 27 additions & 0 deletions src/Polly/CircuitBreaker/BrokenCircuitException.TResult.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ public class BrokenCircuitException<TResult> : BrokenCircuitException
/// <param name="result">The result which caused the circuit to break.</param>
public BrokenCircuitException(TResult result) => Result = result;

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException{TResult}"/> class.
/// </summary>
/// <param name="result">The result which caused the circuit to break.</param>
/// <param name="retryAfter">The time during which the circuit will be broken.</param>
public BrokenCircuitException(TResult result, TimeSpan retryAfter)
: base(retryAfter) => Result = result;

Check warning on line 30 in src/Polly/CircuitBreaker/BrokenCircuitException.TResult.cs

View check run for this annotation

Codecov / codecov/patch

src/Polly/CircuitBreaker/BrokenCircuitException.TResult.cs#L30

Added line #L30 was not covered by tests

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException{TResult}"/> class.
/// </summary>
Expand All @@ -29,6 +37,15 @@ public class BrokenCircuitException<TResult> : BrokenCircuitException
public BrokenCircuitException(string message, TResult result)
: base(message) => Result = result;

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException{TResult}"/> class.
/// </summary>
/// <param name="message">The message that describes the error.</param>
/// <param name="result">The result which caused the circuit to break.</param>
/// <param name="retryAfter">The time during which the circuit will be broken.</param>
public BrokenCircuitException(string message, TResult result, TimeSpan retryAfter)
: base(message, retryAfter) => Result = result;

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException{TResult}"/> class.
/// </summary>
Expand All @@ -38,6 +55,16 @@ public BrokenCircuitException(string message, TResult result)
public BrokenCircuitException(string message, Exception inner, TResult result)
: base(message, inner) => Result = result;

/// <summary>
/// Initializes a new instance of the <see cref="BrokenCircuitException{TResult}"/> class.
/// </summary>
/// <param name="message">The message that describes the error.</param>
/// <param name="inner">The inner exception.</param>
/// <param name="result">The result which caused the circuit to break.</param>
/// <param name="retryAfter">The time during which the circuit will be broken.</param>
public BrokenCircuitException(string message, Exception inner, TResult result, TimeSpan retryAfter)
: base(message, inner, retryAfter) => Result = result;

Check warning on line 66 in src/Polly/CircuitBreaker/BrokenCircuitException.TResult.cs

View check run for this annotation

Codecov / codecov/patch

src/Polly/CircuitBreaker/BrokenCircuitException.TResult.cs#L66

Added line #L66 was not covered by tests

/// <summary>
/// Gets the result value which was considered a handled fault, by the policy.
/// </summary>
Expand Down
8 changes: 4 additions & 4 deletions src/Polly/CircuitBreaker/CircuitStateController.cs
Original file line number Diff line number Diff line change
Expand Up @@ -126,19 +126,19 @@ protected bool PermitHalfOpenCircuitTest()
private BrokenCircuitException GetBreakingException()
{
const string BrokenCircuitMessage = "The circuit is now open and is not allowing calls.";

TimeSpan retryAfter = new TimeSpan(BlockedTill - SystemClock.UtcNow().Ticks);
var lastOutcome = LastOutcome;
if (lastOutcome == null)
{
return new BrokenCircuitException(BrokenCircuitMessage);
return new BrokenCircuitException(BrokenCircuitMessage, retryAfter);

Check warning on line 133 in src/Polly/CircuitBreaker/CircuitStateController.cs

View check run for this annotation

Codecov / codecov/patch

src/Polly/CircuitBreaker/CircuitStateController.cs#L133

Added line #L133 was not covered by tests
}

if (lastOutcome.Exception != null)
{
return new BrokenCircuitException(BrokenCircuitMessage, lastOutcome.Exception);
return new BrokenCircuitException(BrokenCircuitMessage, lastOutcome.Exception, retryAfter);
}

return new BrokenCircuitException<TResult>(BrokenCircuitMessage, lastOutcome.Result);
return new BrokenCircuitException<TResult>(BrokenCircuitMessage, lastOutcome.Result, retryAfter);
}

public void OnActionPreExecute()
Expand Down
4 changes: 3 additions & 1 deletion src/Polly/PublicAPI.Unshipped.txt
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@

Polly.CircuitBreaker.BrokenCircuitException<TResult>.BrokenCircuitException(string message, System.Exception inner, TResult result, System.TimeSpan retryAfter) -> void
Polly.CircuitBreaker.BrokenCircuitException<TResult>.BrokenCircuitException(string message, TResult result, System.TimeSpan retryAfter) -> void
Polly.CircuitBreaker.BrokenCircuitException<TResult>.BrokenCircuitException(TResult result, System.TimeSpan retryAfter) -> void
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,27 @@ public class BrokenCircuitExceptionTests
[Fact]
public void Ctor_Ok()
{
new BrokenCircuitException().Message.Should().Be("The circuit is now open and is not allowing calls.");
new BrokenCircuitException("Dummy.").Message.Should().Be("Dummy.");
new BrokenCircuitException("Dummy.", new InvalidOperationException()).Message.Should().Be("Dummy.");
new BrokenCircuitException("Dummy.", new InvalidOperationException()).InnerException.Should().BeOfType<InvalidOperationException>();
TimeSpan retryAfter = new(1, 0, 0);
BrokenCircuitException defaultException = new();
defaultException.Message.Should().Be("The circuit is now open and is not allowing calls.");
defaultException.RetryAfter.Should().BeNull();
BrokenCircuitException retryAfterException = new(retryAfter);
retryAfterException.Message.Should().Be($"The circuit is now open and is not allowing calls. It can be retried after '{retryAfter}'.");
retryAfterException.RetryAfter.Should().Be(retryAfter);
BrokenCircuitException dummyMessageException = new("Dummy.");
dummyMessageException.Message.Should().Be("Dummy.");
dummyMessageException.RetryAfter.Should().BeNull();
BrokenCircuitException dummyMessageWithRetryAfterException = new("Dummy.", retryAfter);
dummyMessageWithRetryAfterException.Message.Should().Be("Dummy.");
dummyMessageWithRetryAfterException.RetryAfter.Should().Be(retryAfter);
BrokenCircuitException dummyMessageExceptionWithInnerException = new("Dummy.", new InvalidOperationException());
dummyMessageExceptionWithInnerException.Message.Should().Be("Dummy.");
dummyMessageExceptionWithInnerException.InnerException.Should().BeOfType<InvalidOperationException>();
dummyMessageExceptionWithInnerException.RetryAfter.Should().BeNull();
BrokenCircuitException dummyMessageExceptionWithInnerExceptionAndRetryAfter = new("Dummy.", new InvalidOperationException(), retryAfter);
dummyMessageExceptionWithInnerExceptionAndRetryAfter.Message.Should().Be("Dummy.");
dummyMessageExceptionWithInnerExceptionAndRetryAfter.InnerException.Should().BeOfType<InvalidOperationException>();
dummyMessageExceptionWithInnerExceptionAndRetryAfter.RetryAfter.Should().Be(retryAfter);
}

#if !NETCOREAPP
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public void AddCircuitBreaker_IntegrationTest()
int closed = 0;
int halfOpened = 0;

var breakDuration = TimeSpan.FromSeconds(1);
var halfBreakDuration = TimeSpan.FromMilliseconds(500);
var breakDuration = halfBreakDuration + halfBreakDuration;

var options = new CircuitBreakerStrategyOptions
{
Expand All @@ -94,15 +95,25 @@ public void AddCircuitBreaker_IntegrationTest()
opened.Should().Be(1);
halfOpened.Should().Be(0);
closed.Should().Be(0);
Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
BrokenCircuitException exception = Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
exception.RetryAfter.Should().Be(breakDuration);

// Circuit still open after some time
timeProvider.Advance(halfBreakDuration);
opened.Should().Be(1);
halfOpened.Should().Be(0);
closed.Should().Be(0);
exception = Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
exception.RetryAfter.Should().Be(halfBreakDuration);

// Circuit Half Opened
timeProvider.Advance(breakDuration);
timeProvider.Advance(halfBreakDuration);
strategy.Execute(_ => -1);
Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
exception = Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
opened.Should().Be(2);
halfOpened.Should().Be(1);
closed.Should().Be(0);
exception.RetryAfter.Should().Be(breakDuration);

// Now close it
timeProvider.Advance(breakDuration);
Expand All @@ -119,7 +130,8 @@ public void AddCircuitBreaker_IntegrationTest_WithBreakDurationGenerator()
int closed = 0;
int halfOpened = 0;

var breakDuration = TimeSpan.FromSeconds(1);
var halfBreakDuration = TimeSpan.FromMilliseconds(500);
var breakDuration = halfBreakDuration + halfBreakDuration;

var options = new CircuitBreakerStrategyOptions
{
Expand All @@ -146,15 +158,25 @@ public void AddCircuitBreaker_IntegrationTest_WithBreakDurationGenerator()
opened.Should().Be(1);
halfOpened.Should().Be(0);
closed.Should().Be(0);
Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
BrokenCircuitException exception = Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
exception.RetryAfter.Should().Be(breakDuration);

// Circuit still open after some time
timeProvider.Advance(halfBreakDuration);
opened.Should().Be(1);
halfOpened.Should().Be(0);
closed.Should().Be(0);
exception = Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
exception.RetryAfter.Should().Be(halfBreakDuration);

// Circuit Half Opened
timeProvider.Advance(breakDuration);
timeProvider.Advance(halfBreakDuration);
strategy.Execute(_ => -1);
Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
exception = Assert.Throws<BrokenCircuitException>(() => strategy.Execute(_ => 0));
opened.Should().Be(2);
halfOpened.Should().Be(1);
closed.Should().Be(0);
exception.RetryAfter.Should().Be(breakDuration);

// Now close it
timeProvider.Advance(breakDuration);
Expand All @@ -178,8 +200,8 @@ public async Task AddCircuitBreakers_WithIsolatedManualControl_ShouldBeIsolated(
.AddCircuitBreaker(new() { ManualControl = manualControl })
.Build();

strategy1.Invoking(s => s.Execute(() => { })).Should().Throw<IsolatedCircuitException>();
strategy2.Invoking(s => s.Execute(() => { })).Should().Throw<IsolatedCircuitException>();
strategy1.Invoking(s => s.Execute(() => { })).Should().Throw<IsolatedCircuitException>().Where(e => e.RetryAfter == null);
strategy2.Invoking(s => s.Execute(() => { })).Should().Throw<IsolatedCircuitException>().Where(e => e.RetryAfter == null);

await manualControl.CloseAsync();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public async Task Ctor_ManualControl_EnsureAttached()
var strategy = Create();

await _options.ManualControl.IsolateAsync(CancellationToken.None);
strategy.Invoking(s => s.Execute(_ => 0)).Should().Throw<IsolatedCircuitException>();
strategy.Invoking(s => s.Execute(_ => 0)).Should().Throw<IsolatedCircuitException>().Where(e => e.RetryAfter == null);

await _options.ManualControl.CloseAsync(CancellationToken.None);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ public async Task IsolateAsync_Ok()
called.Should().BeTrue();

var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>();
outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>()
.And.Subject.As<IsolatedCircuitException>().RetryAfter.Should().BeNull();

// now close it
await controller.CloseCircuitAsync(ResilienceContextPool.Shared.Get());
Expand Down Expand Up @@ -119,7 +120,7 @@ public async Task OnActionPreExecute_CircuitOpenedByValue()

await OpenCircuit(controller, Outcome.FromResult(99));
var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
error.Should().BeOfType<BrokenCircuitException>();
error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();

GetBlockedTill(controller).Should().Be(_timeProvider.GetUtcNow() + _options.BreakDuration);
}
Expand Down Expand Up @@ -147,6 +148,7 @@ await OpenCircuit(
{
stacks.Add(e.StackTrace!);
e.Message.Should().Be("The circuit is now open and is not allowing calls.");
e.RetryAfter.Should().NotBeNull();

if (innerException)
{
Expand Down Expand Up @@ -206,6 +208,7 @@ public async Task OnActionPreExecute_CircuitOpenedByException()
await OpenCircuit(controller, Outcome.FromException<int>(new InvalidOperationException()));
var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
error.InnerException.Should().BeOfType<InvalidOperationException>();
error.RetryAfter.Should().NotBeNull();
}

[Fact]
Expand Down Expand Up @@ -258,7 +261,7 @@ public async Task OnActionPreExecute_HalfOpen()
// act
await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
var error = (await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception;
error.Should().BeOfType<BrokenCircuitException>();
error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();

// assert
controller.CircuitState.Should().Be(CircuitState.HalfOpen);
Expand Down Expand Up @@ -462,7 +465,7 @@ public async Task OnActionFailureAsync_VoidResult_EnsureBreakingExceptionNotSet(
// assert
controller.LastException.Should().BeNull();
var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>();
outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
}

[Fact]
Expand Down Expand Up @@ -495,9 +498,11 @@ public async Task Flow_Closed_HalfOpen_Open_HalfOpen_Closed()
controller.CircuitState.Should().Be(CircuitState.Open);

// execution rejected
AdvanceTime(TimeSpan.FromMilliseconds(1));
TimeSpan advanceTimeRejected = TimeSpan.FromMilliseconds(1);
AdvanceTime(advanceTimeRejected);
var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>();
outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>()
.And.Subject.As<BrokenCircuitException>().RetryAfter.Should().Be(_options.BreakDuration - advanceTimeRejected);

// wait and try, transition to half open
AdvanceTime(_options.BreakDuration + _options.BreakDuration);
Expand Down
Loading

0 comments on commit 8ede4af

Please sign in to comment.