Availability: Adds account-level read regions as effective preferred regions when preferred regions is not set on client.

Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs

@@ -22,6 +22,7 @@ internal sealed class GatewayAccountReader
         private readonly CosmosHttpClient httpClient;
         private readonly Uri serviceEndpoint;
         private readonly CancellationToken cancellationToken;
+        private readonly ReaderWriterLockSlim readerWriterLock;
 
         // Backlog: Auth abstractions are spilling through. 4 arguments for this CTOR are result of it.
         public GatewayAccountReader(Uri serviceEndpoint,
@@ -35,6 +36,7 @@ public GatewayAccountReader(Uri serviceEndpoint,
             this.cosmosAuthorization = cosmosAuthorization ?? throw new ArgumentNullException(nameof(AuthorizationTokenProvider));
             this.connectionPolicy = connectionPolicy;
             this.cancellationToken = cancellationToken;
+            this.readerWriterLock = new ReaderWriterLockSlim();
         }
 
         private async Task<AccountProperties> GetDatabaseAccountAsync(Uri serviceEndpoint)
@@ -90,7 +92,8 @@ public async Task<AccountProperties> InitializeReaderAsync()
                 locations: this.connectionPolicy.PreferredLocations,
                 accountInitializationCustomEndpoints: this.connectionPolicy.AccountInitializationCustomEndpoints,
                 getDatabaseAccountFn: this.GetDatabaseAccountAsync,
-                cancellationToken: this.cancellationToken);
+                cancellationToken: this.cancellationToken,
+                accountPropertiesReaderWriterLock: this.readerWriterLock);
 
             return databaseAccount;
         }

Microsoft.Azure.Cosmos/src/ReadManyRequestOptions.cs

@@ -68,7 +68,8 @@ internal QueryRequestOptions ConvertToQueryRequestOptions()
                 IfMatchEtag = this.IfMatchEtag,
                 IfNoneMatchEtag = this.IfNoneMatchEtag,
                 Properties = this.Properties,
-                AddRequestHeaders = this.AddRequestHeaders
+                AddRequestHeaders = this.AddRequestHeaders,
+                ExcludeRegions = this.ExcludeRegions
             };
         }
     }

Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInlineCore.cs

@@ -442,7 +442,7 @@ public override Task<ResponseMessage> ReadManyItemsStreamAsync(
                 containerName: this.Id,
                 databaseName: this.Database.Id,
                 operationType: Documents.OperationType.Read,
-                requestOptions: null,
+                requestOptions: readManyRequestOptions,
                 task: (trace) => base.ReadManyItemsStreamAsync(items, trace, readManyRequestOptions, cancellationToken),
                 openTelemetry: new (OpenTelemetryConstants.Operations.ReadManyItems, (response) => new OpenTelemetryResponse(responseMessage: response)));
         }
@@ -457,7 +457,7 @@ public override Task<FeedResponse<T>> ReadManyItemsAsync<T>(
                 containerName: this.Id,
                 databaseName: this.Database.Id,
                 operationType: Documents.OperationType.Read,
-                requestOptions: null,
+                requestOptions: readManyRequestOptions,
                 task: (trace) => base.ReadManyItemsAsync<T>(items, trace, readManyRequestOptions, cancellationToken),
                 openTelemetry: new (OpenTelemetryConstants.Operations.ReadManyItems, (response) => new OpenTelemetryResponse<T>(responseMessage: response)));
         }

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -37,7 +37,8 @@ internal class GlobalEndpointManager : IGlobalEndpointManager
         private readonly TimeSpan MinTimeBetweenAccountRefresh = TimeSpan.FromSeconds(15);
         private readonly int backgroundRefreshLocationTimeIntervalInMS = GlobalEndpointManager.DefaultBackgroundRefreshLocationTimeIntervalInMS;
         private readonly object backgroundAccountRefreshLock = new object();
-        private readonly object isAccountRefreshInProgressLock = new object();
+        private readonly object isAccountRefreshInProgressLock = new object();
+        private readonly ReaderWriterLockSlim locationCacheDatabaseAccountReadWriteLock = new ReaderWriterLockSlim();
         private bool isAccountRefreshInProgress = false;
         private bool isBackgroundAccountRefreshActive = false;
         private DateTime LastBackgroundRefreshUtc = DateTime.MinValue;
@@ -94,8 +95,16 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con
         public ReadOnlyCollection<Uri> AccountReadEndpoints => this.locationCache.AccountReadEndpoints;
 
         public ReadOnlyCollection<Uri> WriteEndpoints => this.locationCache.WriteEndpoints;
+
+        public int PreferredLocationCount
+        {
+            get
+            {
+                Collection<string> effectivePreferredLocations = this.GetEffectivePreferredLocations();
 
-        public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0;
+                return effectivePreferredLocations.Count;
+            }
+        }
 
         public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request)
         {
@@ -119,7 +128,8 @@ public static async Task<AccountProperties> GetDatabaseAccountFromAnyLocationsAs
             IList<string>? locations,
             IList<Uri>? accountInitializationCustomEndpoints,
             Func<Uri, Task<AccountProperties>> getDatabaseAccountFn,
-            CancellationToken cancellationToken)
+            CancellationToken cancellationToken,
+            ReaderWriterLockSlim accountPropertiesReaderWriterLock)
         {
             using (GetAccountPropertiesHelper threadSafeGetAccountHelper = new GetAccountPropertiesHelper(
                defaultEndpoint,
@@ -128,7 +138,7 @@ public static async Task<AccountProperties> GetDatabaseAccountFromAnyLocationsAs
                getDatabaseAccountFn,
                cancellationToken))
             {
-                return await threadSafeGetAccountHelper.GetAccountPropertiesAsync();
+                return await threadSafeGetAccountHelper.GetAccountPropertiesAsync(accountPropertiesReaderWriterLock);
             }
         }
 
@@ -166,15 +176,15 @@ public GetAccountPropertiesHelper(
                     .GetEnumerator();
             }
 
-            public async Task<AccountProperties> GetAccountPropertiesAsync()
+            public async Task<AccountProperties> GetAccountPropertiesAsync(ReaderWriterLockSlim readerWriterLock)
             {
                 // If there are no preferred regions or private endpoints, then just wait for the global endpoint results
                 if (this.LimitToGlobalEndpointOnly)
                 {
-                    return await this.GetOnlyGlobalEndpointAsync();
+                    return await this.GetOnlyGlobalEndpointAsync(readerWriterLock);
                 }
 
-                Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint);
+                Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint, readerWriterLock);
 
                 // Start a timer to start secondary requests in parallel.
                 Task timerTask = Task.Delay(TimeSpan.FromSeconds(5));
@@ -194,8 +204,8 @@ public async Task<AccountProperties> GetAccountPropertiesAsync()
                 HashSet<Task> tasksToWaitOn = new HashSet<Task>
                 {
                     globalEndpointTask,
-                    this.TryGetAccountPropertiesFromAllLocationsAsync(),
-                    this.TryGetAccountPropertiesFromAllLocationsAsync()
+                    this.TryGetAccountPropertiesFromAllLocationsAsync(readerWriterLock),
+                    this.TryGetAccountPropertiesFromAllLocationsAsync(readerWriterLock)
                 };
 
                 while (tasksToWaitOn.Any())
@@ -227,14 +237,14 @@ public async Task<AccountProperties> GetAccountPropertiesAsync()
                 throw new AggregateException(this.TransientExceptions);
             }
 
-            private async Task<AccountProperties> GetOnlyGlobalEndpointAsync()
+            private async Task<AccountProperties> GetOnlyGlobalEndpointAsync(ReaderWriterLockSlim readerWriterLock)
             {
                 if (!this.LimitToGlobalEndpointOnly)
                 {
                     throw new ArgumentException("GetOnlyGlobalEndpointAsync should only be called if there are no other private endpoints or regions");
                 }
 
-                await this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint);
+                await this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint, readerWriterLock);
 
                 if (this.AccountProperties != null)
                 {
@@ -262,7 +272,7 @@ private async Task<AccountProperties> GetOnlyGlobalEndpointAsync()
             /// <summary>
             /// This is done in a thread safe way to allow multiple tasks to iterate over the list of service endpoints.
             /// </summary>
-            private async Task TryGetAccountPropertiesFromAllLocationsAsync()
+            private async Task TryGetAccountPropertiesFromAllLocationsAsync(ReaderWriterLockSlim readerWriterLock)
             {
                 while (this.TryMoveNextServiceEndpointhreadSafe(
                         out Uri? serviceEndpoint))
@@ -274,7 +284,8 @@ private async Task TryGetAccountPropertiesFromAllLocationsAsync()
                     }
 
                     await this.GetAndUpdateAccountPropertiesAsync(
-                        endpoint: serviceEndpoint);
+                        endpoint: serviceEndpoint,
+                        readerWriterLock);
                 }
             }
 
@@ -308,7 +319,7 @@ private bool TryMoveNextServiceEndpointhreadSafe(
                 }
             }
 
-            private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint)
+            private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint, ReaderWriterLockSlim readerWriterLock)
             {
                 try
                 {
@@ -326,8 +337,17 @@ private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint)
 
                     if (databaseAccount != null)
                     {
-                        this.AccountProperties = databaseAccount;
-                        this.CancellationTokenSource.Cancel();
+                        readerWriterLock.EnterWriteLock();
+
+                        try
+                        {
+                            this.AccountProperties = databaseAccount;
+                            this.CancellationTokenSource.Cancel();
+                        }
+                        finally
+                        {
+                            readerWriterLock.ExitWriteLock();
+                        }
                     }
                 }
                 catch (Exception e)
@@ -495,9 +515,18 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account
             if (this.cancellationTokenSource.IsCancellationRequested)
             {
                 return;
-            }
+            }
+
+            this.locationCacheDatabaseAccountReadWriteLock.EnterWriteLock();
 
-            this.locationCache.OnDatabaseAccountRead(databaseAccount);
+            try
+            {
+                this.locationCache.OnDatabaseAccountRead(databaseAccount);
+            }
+            finally
+            {
+                this.locationCacheDatabaseAccountReadWriteLock.ExitWriteLock();
+            }
 
             if (this.isBackgroundAccountRefreshActive)
             {
@@ -647,7 +676,18 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
             try
             {
                 this.LastBackgroundRefreshUtc = DateTime.UtcNow;
-                this.locationCache.OnDatabaseAccountRead(await this.GetDatabaseAccountAsync(true));
+                AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true);
+
+                this.locationCacheDatabaseAccountReadWriteLock.EnterWriteLock();
+
+                try
+                {
+                    this.locationCache.OnDatabaseAccountRead(accountProperties);
+                }
+                finally
+                {
+                    this.locationCacheDatabaseAccountReadWriteLock.ExitWriteLock();
+                }
             }
             catch (Exception ex)
             {
@@ -671,10 +711,11 @@ internal async Task<AccountProperties> GetDatabaseAccountAsync(bool forceRefresh
                               obsoleteValue: null,
                               singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
                                   this.defaultEndpoint,
-                                  this.connectionPolicy.PreferredLocations,
+                                  this.GetEffectivePreferredLocations(),
                                   this.connectionPolicy.AccountInitializationCustomEndpoints,
                                   this.GetDatabaseAccountAsync,
-                                  this.cancellationTokenSource.Token),
+                                  this.cancellationTokenSource.Token,
+                                  this.locationCacheDatabaseAccountReadWriteLock),
                               cancellationToken: this.cancellationTokenSource.Token,
                               forceRefresh: forceRefresh);
 #nullable enable
@@ -689,6 +730,25 @@ private bool SkipRefresh(bool forceRefresh)
             TimeSpan timeSinceLastRefresh = DateTime.UtcNow - this.LastBackgroundRefreshUtc;
             return (this.isAccountRefreshInProgress || this.MinTimeBetweenAccountRefresh > timeSinceLastRefresh)
                 && !forceRefresh;
+        }
+
+        public Collection<string> GetEffectivePreferredLocations()
+        {
+            if (this.connectionPolicy.PreferredLocations != null && this.connectionPolicy.PreferredLocations.Count > 0)
+            {
+                return this.connectionPolicy.PreferredLocations;
+            }
+
+            this.locationCacheDatabaseAccountReadWriteLock.EnterReadLock();
+
+            try
+            {
+                return new Collection<string>(this.locationCache.EffectivePreferredLocations);
+            }
+            finally
+            {
+                this.locationCacheDatabaseAccountReadWriteLock.ExitReadLock();
+            }
         }
     }
 }