Availability: Adds account-level read regions as effective preferred regions when preferred regions is not set on client.

Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs

@@ -22,6 +22,7 @@ internal sealed class GatewayAccountReader
         private readonly CosmosHttpClient httpClient;
         private readonly Uri serviceEndpoint;
         private readonly CancellationToken cancellationToken;
+        private readonly ReaderWriterLockSlim readerWriterLock;
 
         // Backlog: Auth abstractions are spilling through. 4 arguments for this CTOR are result of it.
         public GatewayAccountReader(Uri serviceEndpoint,
@@ -35,6 +36,7 @@ public GatewayAccountReader(Uri serviceEndpoint,
             this.cosmosAuthorization = cosmosAuthorization ?? throw new ArgumentNullException(nameof(AuthorizationTokenProvider));
             this.connectionPolicy = connectionPolicy;
             this.cancellationToken = cancellationToken;
+            this.readerWriterLock = new ReaderWriterLockSlim();
         }
 
         private async Task<AccountProperties> GetDatabaseAccountAsync(Uri serviceEndpoint)
@@ -90,7 +92,8 @@ public async Task<AccountProperties> InitializeReaderAsync()
                 locations: this.connectionPolicy.PreferredLocations,
                 accountInitializationCustomEndpoints: this.connectionPolicy.AccountInitializationCustomEndpoints,
                 getDatabaseAccountFn: this.GetDatabaseAccountAsync,
-                cancellationToken: this.cancellationToken);
+                cancellationToken: this.cancellationToken,
+                accountPropertiesReaderWriterLock: this.readerWriterLock);
 
             return databaseAccount;
         }

Microsoft.Azure.Cosmos/src/ReadManyRequestOptions.cs

@@ -68,7 +68,8 @@ internal QueryRequestOptions ConvertToQueryRequestOptions()
                 IfMatchEtag = this.IfMatchEtag,
                 IfNoneMatchEtag = this.IfNoneMatchEtag,
                 Properties = this.Properties,
-                AddRequestHeaders = this.AddRequestHeaders
+                AddRequestHeaders = this.AddRequestHeaders,
+                ExcludeRegions = this.ExcludeRegions
             };
         }
     }

Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInlineCore.cs

@@ -428,7 +428,7 @@ public override Task<ResponseMessage> ReadManyItemsStreamAsync(
                 containerName: this.Id,
                 databaseName: this.Database.Id,
                 operationType: Documents.OperationType.Read,
-                requestOptions: null,
+                requestOptions: readManyRequestOptions,
                 task: (trace) => base.ReadManyItemsStreamAsync(items, trace, readManyRequestOptions, cancellationToken),
                 openTelemetry: (response) => new OpenTelemetryResponse(responseMessage: response));
         }
@@ -443,7 +443,7 @@ public override Task<FeedResponse<T>> ReadManyItemsAsync<T>(
                 containerName: this.Id,
                 databaseName: this.Database.Id,
                 operationType: Documents.OperationType.Read,
-                requestOptions: null,
+                requestOptions: readManyRequestOptions,
                 task: (trace) => base.ReadManyItemsAsync<T>(items, trace, readManyRequestOptions, cancellationToken),
                 openTelemetry: (response) => new OpenTelemetryResponse<T>(responseMessage: response));
         }

Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs

@@ -206,7 +206,7 @@ private void InitializeClientTelemetry(AccountClientConfiguration clientConfig)
                         connectionMode: this.connectionPolicy.ConnectionMode,
                         authorizationTokenProvider: this.cosmosAuthorization,
                         diagnosticsHelper: DiagnosticsHandlerHelper.GetInstance(),
-                        preferredRegions: this.connectionPolicy.PreferredLocations,
+                        preferredRegions: this.globalEndpointManager.GetEffectivePreferredLocations(),
                         globalEndpointManager: this.globalEndpointManager,
                         endpointUrl: clientConfig.ClientTelemetryConfiguration.Endpoint);
 

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs

@@ -120,7 +120,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
                        },
                    accountInitializationCustomEndpoints: null,
                        getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."),
-                       cancellationTokenSource.Token);
+                       cancellationTokenSource.Token,
+                   new ReaderWriterLockSlim());
 
                 Assert.Fail("Previous call should have failed");
             }
@@ -159,7 +160,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
 
                         throw new Exception("This should never be hit since it should stop after the global endpoint hit the nonretriable exception");
                     },
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
 
                 Assert.Fail("Should throw the UnauthorizedException");
             }
@@ -191,7 +193,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
 
                         throw new Microsoft.Azure.Documents.UnauthorizedException("Mock failed exception");
                     },
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
 
                 Assert.Fail("Should throw the UnauthorizedException");
             }
@@ -222,7 +225,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
 
                         throw new Exception("This should never be hit since it should stop after the global endpoint hit the nonretriable exception");
                     },
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
 
                 Assert.Fail("Should throw the ForbiddenException");
             }
@@ -251,7 +255,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
                         exceptions.Add(exception);
                         throw exception;
                     },
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
 
                 Assert.Fail("Should throw the AggregateException");
             }
@@ -314,7 +319,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync()
                 },
                 accountInitializationCustomEndpoints: null,
                 getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri),
-                cancellationToken: default);
+                cancellationToken: default,
+                new ReaderWriterLockSlim());
 
             Assert.AreEqual(globalEndpointResult, databaseAccount);
             Assert.AreEqual(0, slowPrimaryRegionHelper.FailedEndpointCount);
@@ -337,7 +343,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync()
                     },
                     accountInitializationCustomEndpoints: null,
                     getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri),
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
                 stopwatch.Stop();
 
                 Assert.AreEqual(globalEndpointResult, databaseAccount);
@@ -361,7 +368,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync()
                     },
                     accountInitializationCustomEndpoints: null,
                     getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri),
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
 
                 Assert.AreEqual(globalEndpointResult, databaseAccount);
                 Assert.AreEqual(3, slowPrimaryRegionHelper.FailedEndpointCount);
@@ -383,7 +391,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync()
                     },
                     accountInitializationCustomEndpoints: null,
                     getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri),
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
 
                 Assert.AreEqual(globalEndpointResult, databaseAccount);
                 Assert.AreEqual(0, slowPrimaryRegionHelper.FailedEndpointCount);
@@ -409,14 +418,79 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync()
                     },
                     accountInitializationCustomEndpoints: null,
                     getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri),
-                    cancellationToken: default);
+                    cancellationToken: default,
+                    new ReaderWriterLockSlim());
 
                 Assert.AreEqual(globalEndpointResult, databaseAccount);
                 Assert.AreEqual(5, slowPrimaryRegionHelper.FailedEndpointCount);
                 Assert.IsTrue(slowPrimaryRegionHelper.ReturnedSuccess);
             }
         }
 
+        /// <summary>
+        /// Test to validate for a client that has been warmed up with account-level regions, any subsequent
+        /// DatabaseAccount refresh calls should go through the effective preferred regions / account-level read regions
+        /// if the DatabaseAccount refresh call to the global / default endpoint failed with HttpRequestException (timeout also but not possible to inject
+        /// w/o adding a refresh method just for this test)
+        /// </summary>
+        [TestMethod]
+        public async Task GetDatabaseAccountFromEffectiveRegionalEndpointTestAsync()
+        {
+            AccountProperties databaseAccount = new AccountProperties
+            {
+                ReadLocationsInternal = new Collection<AccountRegion>()
+                {
+                    new AccountRegion
+                    {
+                        Name = "Location1",
+                        Endpoint = "https://testfailover-location1.documents-test.windows-int.net/"
+                    },
+                    new AccountRegion
+                    {
+                        Name = "Location2",
+                        Endpoint = "https://testfailover-location2.documents-test.windows-int.net/"
+                    },
+                    new AccountRegion
+                    {
+                        Name = "Location3",
+                        Endpoint = "https://testfailover-location3.documents-test.windows-int.net/"
+                    },
+                }
+            };
+
+            Uri defaultEndpoint = new Uri("https://testfailover.documents-test.windows-int.net/");
+            Uri effectivePreferredRegion1SuffixedUri = new Uri("https://testfailover-location1.documents-test.windows-int.net/");
+
+            //Setup mock owner "document client"
+            Mock<IDocumentClientInternal> mockOwner = new Mock<IDocumentClientInternal>();
+
+            mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(defaultEndpoint);
+            mockOwner.SetupSequence(owner =>
+                    owner.GetDatabaseAccountInternalAsync(defaultEndpoint, It.IsAny<CancellationToken>()))
+                .ReturnsAsync(databaseAccount)
+                .ThrowsAsync(new HttpRequestException());
+            mockOwner.Setup(owner =>
+                    owner.GetDatabaseAccountInternalAsync(effectivePreferredRegion1SuffixedUri, It.IsAny<CancellationToken>()))
+                .ReturnsAsync(databaseAccount);
+
+            // Create connection policy with no preferred locations
+            ConnectionPolicy connectionPolicy = new ConnectionPolicy();
+
+            using GlobalEndpointManager globalEndpointManager =
+                new GlobalEndpointManager(mockOwner.Object, connectionPolicy);
+            globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount);
+
+            await Task.Delay(TimeSpan.FromSeconds(5));
+            await globalEndpointManager.RefreshLocationAsync(forceRefresh: true);
+
+            mockOwner.Verify(
+                owner => owner.GetDatabaseAccountInternalAsync(defaultEndpoint, It.IsAny<CancellationToken>()),
+                Times.Exactly(2));
+            mockOwner.Verify(
+                owner => owner.GetDatabaseAccountInternalAsync(effectivePreferredRegion1SuffixedUri, It.IsAny<CancellationToken>()),
+                Times.Once);
+        }
+
         /// <summary>
         /// Test to validate that when an exception is thrown during a RefreshLocationAsync call
         /// the exception should not be bubbled up and remain unobserved. The exception should be