diff --git a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs index be166ed5a9..69eddb2fbc 100644 --- a/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs +++ b/Microsoft.Azure.Cosmos/src/Batch/BatchAsyncContainerExecutor.cs @@ -92,7 +92,11 @@ public virtual async Task AddAsync( ItemBatchOperationContext context = new ItemBatchOperationContext( resolvedPartitionKeyRangeId, trace, - BatchAsyncContainerExecutor.GetRetryPolicy(this.cosmosContainer, operation.OperationType, this.retryOptions)); + BatchAsyncContainerExecutor.GetRetryPolicy( + this.cosmosContainer, + this.cosmosClientContext?.DocumentClient?.GlobalEndpointManager, + operation.OperationType, + this.retryOptions)); if (itemRequestOptions != null && itemRequestOptions.AddRequestHeaders != null) { @@ -159,6 +163,7 @@ internal virtual async Task ValidateOperationAsync( private static IDocumentClientRetryPolicy GetRetryPolicy( ContainerInternal containerInternal, + GlobalEndpointManager endpointManager, OperationType operationType, RetryOptions retryOptions) { @@ -167,6 +172,7 @@ private static IDocumentClientRetryPolicy GetRetryPolicy( operationType, new ResourceThrottleRetryPolicy( retryOptions.MaxRetryAttemptsOnThrottledRequests, + endpointManager, retryOptions.MaxRetryWaitTimeInSeconds)); } diff --git a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs index e19433c17e..9e04c8a8c1 100644 --- a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs @@ -47,8 +47,9 @@ public ClientRetryPolicy( bool isPertitionLevelFailoverEnabled) { this.throttlingRetry = new ResourceThrottleRetryPolicy( - retryOptions.MaxRetryAttemptsOnThrottledRequests, - retryOptions.MaxRetryWaitTimeInSeconds); + maxAttemptCount: retryOptions.MaxRetryAttemptsOnThrottledRequests, + endpointManager: globalEndpointManager, + maxWaitTimeInSeconds: retryOptions.MaxRetryWaitTimeInSeconds); this.globalEndpointManager = globalEndpointManager; this.partitionKeyRangeLocationCache = partitionKeyRangeLocationCache; diff --git a/Microsoft.Azure.Cosmos/src/DocumentClient.cs b/Microsoft.Azure.Cosmos/src/DocumentClient.cs index ce1c9a43c2..c0672b6290 100644 --- a/Microsoft.Azure.Cosmos/src/DocumentClient.cs +++ b/Microsoft.Azure.Cosmos/src/DocumentClient.cs @@ -985,6 +985,7 @@ internal virtual void Initialize(Uri serviceEndpoint, () => this.GetInitializationTaskAsync(storeClientFactory: storeClientFactory), new ResourceThrottleRetryPolicy( this.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests, + this.GlobalEndpointManager, this.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds)); // Create the task to start the initialize task diff --git a/Microsoft.Azure.Cosmos/src/MetadataRequestThrottleRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/MetadataRequestThrottleRetryPolicy.cs index 928d2f2e87..7d0d6649fe 100644 --- a/Microsoft.Azure.Cosmos/src/MetadataRequestThrottleRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/MetadataRequestThrottleRetryPolicy.cs @@ -72,6 +72,7 @@ public MetadataRequestThrottleRetryPolicy( this.throttlingRetryPolicy = new ResourceThrottleRetryPolicy( maxRetryAttemptsOnThrottledRequests, + globalEndpointManager, maxRetryWaitTimeInSeconds); this.retryContext = new MetadataRetryContext diff --git a/Microsoft.Azure.Cosmos/src/ResourceThrottleRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/ResourceThrottleRetryPolicy.cs index a99bc594f7..94da0dad65 100644 --- a/Microsoft.Azure.Cosmos/src/ResourceThrottleRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ResourceThrottleRetryPolicy.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Cosmos using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Documents; // Retry when we receive the throttling from server. @@ -19,12 +20,15 @@ internal sealed class ResourceThrottleRetryPolicy : IDocumentClientRetryPolicy private readonly uint backoffDelayFactor; private readonly int maxAttemptCount; private readonly TimeSpan maxWaitTimeInMilliseconds; + private readonly IGlobalEndpointManager globalEndpointManager; private int currentAttemptCount; private TimeSpan cumulativeRetryDelay; + private bool? isMultiMasterWriteRegion; public ResourceThrottleRetryPolicy( int maxAttemptCount, + IGlobalEndpointManager endpointManager, int maxWaitTimeInSeconds = DefaultMaxWaitTimeInSeconds, uint backoffDelayFactor = 1) { @@ -33,6 +37,7 @@ public ResourceThrottleRetryPolicy( throw new ArgumentException("maxWaitTimeInSeconds", "maxWaitTimeInSeconds must be less than " + (int.MaxValue / 1000)); } + this.globalEndpointManager = endpointManager; this.maxAttemptCount = maxAttemptCount; this.backoffDelayFactor = backoffDelayFactor; this.maxWaitTimeInMilliseconds = TimeSpan.FromSeconds(maxWaitTimeInSeconds); @@ -59,7 +64,9 @@ public Task ShouldRetryAsync( return Task.FromResult(ShouldRetryResult.NoRetry()); } - return this.ShouldRetryInternalAsync(dce.RetryAfter); + return this.ShouldRetryInternalAsync( + dce?.GetSubStatus(), + dce?.RetryAfter); } DefaultTrace.TraceError( @@ -88,11 +95,34 @@ public Task ShouldRetryAsync( return Task.FromResult(ShouldRetryResult.NoRetry()); } - return this.ShouldRetryInternalAsync(cosmosResponseMessage?.Headers.RetryAfter); + return this.ShouldRetryInternalAsync( + cosmosResponseMessage?.Headers.SubStatusCode, + cosmosResponseMessage?.Headers.RetryAfter, + cosmosResponseMessage?.CosmosException); } - private Task ShouldRetryInternalAsync(TimeSpan? retryAfter) + private Task ShouldRetryInternalAsync( + SubStatusCodes? subStatusCode, + TimeSpan? retryAfter, + Exception exception = null) { + if (this.isMultiMasterWriteRegion.HasValue + && this.isMultiMasterWriteRegion.Value + && subStatusCode != null + && subStatusCode == SubStatusCodes.SystemResourceUnavailable) + { + DefaultTrace.TraceError( + "Operation will NOT be retried. Converting 429 to 503. Current attempt {0} sub status code: {1}.", + this.currentAttemptCount, SubStatusCodes.SystemResourceUnavailable); + + var exceptionToThrow = ServiceUnavailableException.Create( + SubStatusCodes.SystemResourceUnavailable, + innerException: exception); + + return Task.FromResult( + ShouldRetryResult.NoRetry(exceptionToThrow)); + } + TimeSpan retryDelay = TimeSpan.Zero; if (this.currentAttemptCount < this.maxAttemptCount && this.CheckIfRetryNeeded(retryAfter, out retryDelay)) @@ -133,6 +163,8 @@ private object GetExceptionMessage(Exception exception) /// The request being sent to the service. public void OnBeforeSendRequest(DocumentServiceRequest request) { + this.isMultiMasterWriteRegion = !request.IsReadOnlyRequest + && (this.globalEndpointManager?.CanUseMultipleWriteLocations(request) ?? false); } /// diff --git a/Microsoft.Azure.Cosmos/src/direct/GoneAndRetryWithRequestRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/direct/GoneAndRetryWithRequestRetryPolicy.cs index 91d2e6392a..0eac20b794 100644 --- a/Microsoft.Azure.Cosmos/src/direct/GoneAndRetryWithRequestRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/direct/GoneAndRetryWithRequestRetryPolicy.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Documents using System.Net; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; using Microsoft.Azure.Cosmos.Core.Trace; using Microsoft.Azure.Documents.Routing; @@ -143,6 +144,7 @@ public bool TryHandleResponseSynchronously(DocumentServiceRequest request, TResp bool isRetryWith = false; if (!GoneAndRetryWithRequestRetryPolicy.IsBaseGone(response, exception) && + !GoneAndRetryWithRequestRetryPolicy.IsGoneWithLeaseNotFound(response, exception) && !(exception is RetryWithException) && !(GoneAndRetryWithRequestRetryPolicy.IsPartitionIsMigrating(response, exception) && (request.ServiceIdentity == null || request.ServiceIdentity.IsMasterService)) && !(GoneAndRetryWithRequestRetryPolicy.IsInvalidPartition(response, exception) && (request.PartitionKeyRangeIdentity == null || request.PartitionKeyRangeIdentity.CollectionRid == null)) && @@ -170,6 +172,21 @@ public bool TryHandleResponseSynchronously(DocumentServiceRequest request, TResp isRetryWith = true; this.lastRetryWithException = exception as RetryWithException; } + else if (GoneAndRetryWithRequestRetryPolicy.IsGoneWithLeaseNotFound(response, exception)) + { + DefaultTrace.TraceWarning( + "The GoneAndRetryWithRequestRetryPolicy has hit 410 with lease not found. This is by design to do a cross regional failover to handle the exception: {0}", + new ErrorOrResponse(exception)); + + exceptionToThrow = ServiceUnavailableException.Create( + SubStatusCodes.LeaseNotFound, + innerException: exception); + + this.durationTimer.Stop(); + + shouldRetryResult = ShouldRetryResult.NoRetry(exceptionToThrow); + return true; + } int remainingMilliseconds; if (isRetryWith) @@ -428,6 +445,13 @@ private static bool IsPartitionKeyRangeGone(TResponse response, Exception except || (response?.StatusCode == HttpStatusCode.Gone && response?.SubStatusCode == SubStatusCodes.PartitionKeyRangeGone); } + private static bool IsGoneWithLeaseNotFound(TResponse response, Exception exception) + { + return exception is LeaseLostException + || (response?.StatusCode == HttpStatusCode.Gone && + (response?.SubStatusCode == SubStatusCodes.LeaseNotFound)); + } + private static void ClearRequestContext(DocumentServiceRequest request) { request.RequestContext.TargetIdentity = null; diff --git a/Microsoft.Azure.Cosmos/src/direct/StatusCodes.cs b/Microsoft.Azure.Cosmos/src/direct/StatusCodes.cs index 866c5abb2e..b8358467f1 100644 --- a/Microsoft.Azure.Cosmos/src/direct/StatusCodes.cs +++ b/Microsoft.Azure.Cosmos/src/direct/StatusCodes.cs @@ -212,6 +212,7 @@ internal enum SubStatusCodes GatewayThrottled = 3201, StoredProcedureConcurrency = 3084, ThottleDueToSplit = 3088, + SystemResourceUnavailable = 3092, // Key Vault Access Client Error Code AadClientCredentialsGrantFailure = 4000, // Indicated access to AAD failed to get a token diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs index acc83d3ff4..2d323039f5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncBatcherTests.cs @@ -562,15 +562,18 @@ public async Task CannotAddToDispatchedBatch() [TestMethod] public async Task RetrierGetsCalledOnSplit() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); ItemBatchOperation operation2 = this.CreateItemBatchOperation(); @@ -591,15 +594,18 @@ public async Task RetrierGetsCalledOnSplit() [TestMethod] public async Task RetrierGetsCalledOnCompletingSplit() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); ItemBatchOperation operation2 = this.CreateItemBatchOperation(); @@ -620,15 +626,18 @@ public async Task RetrierGetsCalledOnCompletingSplit() [TestMethod] public async Task RetrierGetsCalledOnCompletingPartitionMigration() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); ItemBatchOperation operation2 = this.CreateItemBatchOperation(); @@ -669,20 +678,23 @@ public async Task RetrierGetsCalledOnOverFlow() [TestMethod] public async Task RetrierGetsCalledOn413_3402() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); IDocumentClientRetryPolicy retryPolicy3 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Create, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); ItemBatchOperation operation2 = this.CreateItemBatchOperation(); @@ -707,20 +719,23 @@ public async Task RetrierGetsCalledOn413_3402() [TestMethod] public async Task RetrierGetsCalledOn413_NoSubstatus() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy1 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); IDocumentClientRetryPolicy retryPolicy2 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); IDocumentClientRetryPolicy retryPolicy3 = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Create, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); ItemBatchOperation operation1 = this.CreateItemBatchOperation(); ItemBatchOperation operation2 = this.CreateItemBatchOperation(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs index 8e3c6bcbc3..ebf7f86dce 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Batch/BatchAsyncOperationContextTests.cs @@ -21,10 +21,13 @@ public class BatchAsyncOperationContextTests [TestMethod] public async Task TraceIsJoinedOnCompletionWithRetry() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); Trace rootTrace = Trace.GetRootTrace(name: "RootTrace"); @@ -62,10 +65,13 @@ public async Task TraceIsJoinedOnCompletionWithRetry() [TestMethod] public async Task TraceIsJoinedOnCompletionWithoutRetry() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); Trace rootTrace = Trace.GetRootTrace(name: "RootTrace"); @@ -176,10 +182,13 @@ public async Task ShouldRetry_NoPolicy() [TestMethod] public async Task ShouldRetry_WithPolicy_OnSuccess() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.OK); ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); operation.AttachContext(new ItemBatchOperationContext(string.Empty, NoOpTrace.Singleton, retryPolicy)); @@ -190,10 +199,13 @@ public async Task ShouldRetry_WithPolicy_OnSuccess() [TestMethod] public async Task ShouldRetry_WithPolicy_On429() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests); ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); operation.AttachContext(new ItemBatchOperationContext(string.Empty, NoOpTrace.Singleton, retryPolicy)); @@ -204,10 +216,13 @@ public async Task ShouldRetry_WithPolicy_On429() [TestMethod] public async Task ShouldRetry_WithPolicy_On413_3402() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge) { SubStatusCode = (SubStatusCodes)3402 }; ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); operation.AttachContext(new ItemBatchOperationContext(string.Empty, NoOpTrace.Singleton, retryPolicy)); @@ -218,10 +233,13 @@ public async Task ShouldRetry_WithPolicy_On413_3402() [TestMethod] public async Task ShouldRetry_WithPolicy_On413_0() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Create, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge); ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); operation.AttachContext(new ItemBatchOperationContext(string.Empty, NoOpTrace.Singleton, retryPolicy)); @@ -232,10 +250,13 @@ public async Task ShouldRetry_WithPolicy_On413_0() [TestMethod] public async Task ShouldRetry_WithPolicy_OnSplit() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); operation.AttachContext(new ItemBatchOperationContext(string.Empty, NoOpTrace.Singleton, retryPolicy)); @@ -246,10 +267,13 @@ public async Task ShouldRetry_WithPolicy_OnSplit() [TestMethod] public async Task ShouldRetry_WithPolicy_OnCompletingSplit() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingSplit }; ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); operation.AttachContext(new ItemBatchOperationContext(string.Empty, NoOpTrace.Singleton, retryPolicy)); @@ -260,10 +284,13 @@ public async Task ShouldRetry_WithPolicy_OnCompletingSplit() [TestMethod] public async Task ShouldRetry_WithPolicy_OnCompletingPartitionMigration() { + Mock mockedClient = new Mock(); + GlobalEndpointManager endpointManager = new GlobalEndpointManager(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingPartitionMigration }; ItemBatchOperation operation = new ItemBatchOperation(OperationType.Create, 0, Cosmos.PartitionKey.Null); operation.AttachContext(new ItemBatchOperationContext(string.Empty, NoOpTrace.Singleton, retryPolicy)); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs index 1f87d40635..2e5110288d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BulkPartitionKeyRangeGoneRetryPolicyTests.cs @@ -21,10 +21,13 @@ public class BulkPartitionKeyRangeGoneRetryPolicyTests [TestMethod] public async Task NotRetryOnSuccess() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.OK); ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); @@ -34,10 +37,13 @@ public async Task NotRetryOnSuccess() [TestMethod] public async Task RetriesOn429() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult((HttpStatusCode)StatusCodes.TooManyRequests); ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); @@ -47,10 +53,13 @@ public async Task RetriesOn429() [TestMethod] public async Task RetriesOn413_3204() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge) { SubStatusCode = (SubStatusCodes)3402 }; ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); @@ -60,10 +69,13 @@ public async Task RetriesOn413_3204() [TestMethod] public async Task RetriesOn413_0() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( Mock.Of(), OperationType.Create, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.RequestEntityTooLarge); ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); @@ -73,10 +85,13 @@ public async Task RetriesOn413_0() [TestMethod] public async Task RetriesOnSplits() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); @@ -86,10 +101,13 @@ public async Task RetriesOnSplits() [TestMethod] public async Task RetriesOnSplits_UpToMax() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.PartitionKeyRangeGone }; ShouldRetryResult shouldRetryResult; @@ -106,10 +124,13 @@ public async Task RetriesOnSplits_UpToMax() [TestMethod] public async Task RetriesOnCompletingSplits() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingSplit }; ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); @@ -119,10 +140,13 @@ public async Task RetriesOnCompletingSplits() [TestMethod] public async Task RetriesOnCompletingPartitionMigrationSplits() { + Mock mockedClient = new(); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + IDocumentClientRetryPolicy retryPolicy = new BulkExecutionRetryPolicy( GetSplitEnabledContainer(), OperationType.Read, - new ResourceThrottleRetryPolicy(1)); + new ResourceThrottleRetryPolicy(1, endpointManager)); TransactionalBatchOperationResult result = new TransactionalBatchOperationResult(HttpStatusCode.Gone) { SubStatusCode = SubStatusCodes.CompletingPartitionMigration }; ShouldRetryResult shouldRetryResult = await retryPolicy.ShouldRetryAsync(result.ToResponseMessage(), default); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ResourceThrottleRetryPolicyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ResourceThrottleRetryPolicyTests.cs index 274bbcb8e9..b58c0605f7 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ResourceThrottleRetryPolicyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ResourceThrottleRetryPolicyTests.cs @@ -9,7 +9,9 @@ namespace Microsoft.Azure.Cosmos.Tests using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Cosmos.Routing; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; [TestClass] public class ResourceThrottleRetryPolicyTests @@ -44,8 +46,11 @@ public void ResetTraceConfiguration() [TestMethod] public async Task DoesNotSerializeExceptionOnTracingDisabled() { + Mock mockedClient = new (); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + // No listeners - ResourceThrottleRetryPolicy policy = new ResourceThrottleRetryPolicy(0); + ResourceThrottleRetryPolicy policy = new ResourceThrottleRetryPolicy(0, endpointManager); CustomException exception = new CustomException(); await policy.ShouldRetryAsync(exception, default); Assert.AreEqual(0, exception.ToStringCount, "Exception was serialized"); @@ -54,10 +59,13 @@ public async Task DoesNotSerializeExceptionOnTracingDisabled() [TestMethod] public async Task DoesSerializeExceptionOnTracingEnabled() { + Mock mockedClient = new (); + GlobalEndpointManager endpointManager = new(mockedClient.Object, new ConnectionPolicy()); + // Let the default trace listener DefaultTrace.TraceSource.Switch = new SourceSwitch("ClientSwitch", "Error"); DefaultTrace.TraceSource.Listeners.Add(new DefaultTraceListener()); - ResourceThrottleRetryPolicy policy = new ResourceThrottleRetryPolicy(0); + ResourceThrottleRetryPolicy policy = new ResourceThrottleRetryPolicy(0, endpointManager); CustomException exception = new CustomException(); await policy.ShouldRetryAsync(exception, default); Assert.AreEqual(1, exception.ToStringCount, "Exception was not serialized");