From 4597f2de7620033e57f0b090f6ebd0686480ed2e Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 9 Sep 2024 13:58:29 -0400 Subject: [PATCH 01/16] Initial changes to LocationCache. --- Microsoft.Azure.Cosmos.sln | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index d412905195..03a17e2f39 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29123.88 +# Visual Studio Version 17 +VisualStudioVersion = 17.10.35201.131 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -164,6 +164,18 @@ Global {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|Any CPU.Build.0 = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.ActiveCfg = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 4b2a4554df6064150a976bf4a39e791ec08461e0 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 9 Sep 2024 14:01:44 -0400 Subject: [PATCH 02/16] Intial changes to LocationCache. --- .../src/Routing/LocationCache.cs | 219 +++++++++++++----- 1 file changed, 161 insertions(+), 58 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index c95223f044..6734d68451 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -8,8 +8,9 @@ namespace Microsoft.Azure.Cosmos.Routing using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; - using System.Linq; - using Microsoft.Azure.Cosmos.Core.Trace; + using System.Linq; + using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Cosmos.Linq; using Microsoft.Azure.Documents; /// @@ -320,48 +321,84 @@ public Uri ResolveServiceEndpoint(DocumentServiceRequest request) public ReadOnlyCollection GetApplicableEndpoints(DocumentServiceRequest request, bool isReadRequest) { - ReadOnlyCollection endpoints = - isReadRequest - ? this.ReadEndpoints - : this.WriteEndpoints; - if (request.RequestContext.ExcludeRegions == null || request.RequestContext.ExcludeRegions.Count == 0) { - return endpoints; + return isReadRequest ? this.ReadEndpoints : this.WriteEndpoints; + } + + DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; + + ReadOnlyCollection effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.PreferredLocations; + + if (effectivePreferredLocations == null || effectivePreferredLocations.Count == 0) + { + effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations; } return this.GetApplicableEndpoints( isReadRequest ? this.locationInfo.AvailableReadEndpointByLocation : this.locationInfo.AvailableWriteEndpointByLocation, + effectivePreferredLocations, this.defaultEndpoint, request.RequestContext.ExcludeRegions); } public ReadOnlyCollection GetApplicableRegions(IEnumerable excludeRegions, bool isReadRequest) - { - return this.GetApplicableRegions( - isReadRequest ? this.locationInfo.AvailableReadLocations : this.locationInfo.AvailableWriteLocations, - this.locationInfo.PreferredLocations[0], - excludeRegions); - } - + { + bool isPreferredLocationsEmpty = this.locationInfo.PreferredLocations == null || this.locationInfo.PreferredLocations.Count == 0; + + DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; + + ReadOnlyCollection effectivePreferredLocations = this.locationInfo.PreferredLocations; + + if (effectivePreferredLocations == null || effectivePreferredLocations.Count == 0) + { + effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations; + } + + Uri firstEffectivePreferredEndpoint = isReadRequest ? databaseAccountLocationsInfoSnapshot.ReadEndpoints[0] : databaseAccountLocationsInfoSnapshot.WriteEndpoints[0]; + + if (isReadRequest) + { + databaseAccountLocationsInfoSnapshot.AvailableReadLocationByEndpoint.TryGetValue(firstEffectivePreferredEndpoint, out string firstEffectivePreferredLocation); + + return this.GetApplicableRegions( + databaseAccountLocationsInfoSnapshot.AvailableReadLocations, + effectivePreferredLocations, + isPreferredLocationsEmpty ? firstEffectivePreferredLocation : databaseAccountLocationsInfoSnapshot.PreferredLocations[0], + excludeRegions); + } + else + { + databaseAccountLocationsInfoSnapshot.AvailableWriteLocationByEndpoint.TryGetValue(firstEffectivePreferredEndpoint, out string firstEffectivePreferredLocation); + + return this.GetApplicableRegions( + databaseAccountLocationsInfoSnapshot.AvailableWriteLocations, + effectivePreferredLocations, + isPreferredLocationsEmpty ? firstEffectivePreferredLocation : databaseAccountLocationsInfoSnapshot.PreferredLocations[0], + excludeRegions); + } + } + /// /// Gets applicable endpoints for a request, if there are no applicable endpoints, returns the fallback endpoint /// - /// + /// + /// /// /// /// a list of applicable endpoints for a request private ReadOnlyCollection GetApplicableEndpoints( - ReadOnlyDictionary regionNameByEndpoint, + ReadOnlyDictionary regionNameByEndpoint, + ReadOnlyCollection effectivePreferredLocations, Uri fallbackEndpoint, IEnumerable excludeRegions) { List applicableEndpoints = new List(regionNameByEndpoint.Count); HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); - + if (excludeRegions != null) { - foreach (string region in this.locationInfo.PreferredLocations) + foreach (string region in effectivePreferredLocations) { if (!excludeRegionsHash.Contains(region) && regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) @@ -372,7 +409,7 @@ private ReadOnlyCollection GetApplicableEndpoints( } else { - foreach (string region in this.locationInfo.PreferredLocations) + foreach (string region in effectivePreferredLocations) { if (regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) { @@ -392,23 +429,25 @@ private ReadOnlyCollection GetApplicableEndpoints( /// /// Gets applicable endpoints for a request, if there are no applicable endpoints, returns the fallback endpoint /// - /// + /// + /// /// /// /// a list of applicable endpoints for a request private ReadOnlyCollection GetApplicableRegions( - ReadOnlyCollection regionNameByEndpoint, + ReadOnlyCollection availableLocations, + ReadOnlyCollection effectivePreferredLocations, string fallbackRegion, IEnumerable excludeRegions) { - List applicableRegions = new List(regionNameByEndpoint.Count); + List applicableRegions = new List(availableLocations.Count); HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); - + if (excludeRegions != null) { - foreach (string region in this.locationInfo.PreferredLocations) + foreach (string region in effectivePreferredLocations) { - if (regionNameByEndpoint.Contains(region) + if (availableLocations.Contains(region) && !excludeRegionsHash.Contains(region)) { applicableRegions.Add(region); @@ -417,9 +456,9 @@ private ReadOnlyCollection GetApplicableRegions( } else { - foreach (string region in this.locationInfo.PreferredLocations) + foreach (string region in effectivePreferredLocations) { - if (regionNameByEndpoint.Contains(region)) + if (availableLocations.Contains(region)) { applicableRegions.Add(region); } @@ -439,8 +478,8 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) canRefreshInBackground = true; DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; - string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); - + string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); + // we should schedule refresh in background if we are unable to target the user's most preferredLocation. if (this.enableEndpointDiscovery) { @@ -471,8 +510,8 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) // other available read endpoints DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not available for read.", mostPreferredLocation); return true; - } - } + } + } else { DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available read locations.", mostPreferredLocation); @@ -507,7 +546,7 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) if (currentLocationInfo.AvailableWriteEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredWriteEndpoint)) { shouldRefresh |= mostPreferredWriteEndpoint != writeLocationEndpoints[0]; - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation); + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation); return shouldRefresh; } else @@ -643,19 +682,23 @@ private void UpdateLocationCache( { nextLocationInfo.AvailableReadEndpointByLocation = this.GetEndpointByLocation( readLocations, - out ReadOnlyCollection availableReadLocations); + out ReadOnlyCollection availableReadLocations, + out ReadOnlyDictionary availableReadLocationsByEndpoint); nextLocationInfo.AvailableReadLocations = availableReadLocations; nextLocationInfo.AccountReadEndpoints = nextLocationInfo.AvailableReadEndpointByLocation.Select(x => x.Value).ToList().AsReadOnly(); + nextLocationInfo.AvailableReadLocationByEndpoint = availableReadLocationsByEndpoint; } if (writeLocations != null) { nextLocationInfo.AvailableWriteEndpointByLocation = this.GetEndpointByLocation( writeLocations, - out ReadOnlyCollection availableWriteLocations); + out ReadOnlyCollection availableWriteLocations, + out ReadOnlyDictionary availableWriteLocationsByEndpoint); nextLocationInfo.AvailableWriteLocations = availableWriteLocations; + nextLocationInfo.AvailableWriteLocationByEndpoint = availableWriteLocationsByEndpoint; } nextLocationInfo.WriteEndpoints = this.GetPreferredAvailableEndpoints( @@ -668,7 +711,24 @@ private void UpdateLocationCache( endpointsByLocation: nextLocationInfo.AvailableReadEndpointByLocation, orderedLocations: nextLocationInfo.AvailableReadLocations, expectedAvailableOperation: OperationType.Read, - fallbackEndpoint: nextLocationInfo.WriteEndpoints[0]); + fallbackEndpoint: nextLocationInfo.WriteEndpoints[0]); + + if (preferenceList == null || preferenceList.Count == 0) + { + if (!nextLocationInfo.AvailableReadLocationByEndpoint.TryGetValue(this.defaultEndpoint, out string regionForDefaultEndpoint)) + { + nextLocationInfo.EffectivePreferredLocations = nextLocationInfo.AvailableReadLocations; + } + else + { + List locations = new () + { + regionForDefaultEndpoint + }; + + nextLocationInfo.EffectivePreferredLocations = new ReadOnlyCollection(locations); + } + } this.lastCacheUpdateTimestamp = DateTime.UtcNow; @@ -697,19 +757,45 @@ private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionar // If client can use multiple write locations, preferred locations list should be used for determining // both read and write endpoints order. - foreach (string location in currentLocationInfo.PreferredLocations) - { - if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) - { - if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) - { - unavailableEndpoints.Add(endpoint); - } - else - { - endpoints.Add(endpoint); - } - } + if (currentLocationInfo.PreferredLocations != null && currentLocationInfo.PreferredLocations.Count >= 1) + { + foreach (string location in currentLocationInfo.PreferredLocations) + { + if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) + { + if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + { + unavailableEndpoints.Add(endpoint); + } + else + { + endpoints.Add(endpoint); + } + } + } + } + else + { + foreach (string location in orderedLocations) + { + if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) + { + if (this.defaultEndpoint.Equals(endpoint)) + { + endpoints = new List(); + break; + } + + if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + { + unavailableEndpoints.Add(endpoint); + } + else + { + endpoints.Add(endpoint); + } + } + } } if (endpoints.Count == 0) @@ -741,9 +827,11 @@ private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionar return endpoints.AsReadOnly(); } - private ReadOnlyDictionary GetEndpointByLocation(IEnumerable locations, out ReadOnlyCollection orderedLocations) + private ReadOnlyDictionary GetEndpointByLocation(IEnumerable locations, out ReadOnlyCollection orderedLocations, out ReadOnlyDictionary availableLocationsByEndpoint) { - Dictionary endpointsByLocation = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary endpointsByLocation = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary mutableAvailableLocationsByEndpoint = new Dictionary(); + List parsedLocations = new List(); foreach (AccountRegion location in locations) @@ -753,7 +841,10 @@ private ReadOnlyDictionary GetEndpointByLocation(IEnumerable GetEndpointByLocation(IEnumerable(mutableAvailableLocationsByEndpoint); + return new ReadOnlyDictionary(endpointsByLocation); } @@ -795,10 +888,13 @@ public DatabaseAccountLocationsInfo(ReadOnlyCollection preferredLocation this.AvailableWriteLocations = new List().AsReadOnly(); this.AvailableReadLocations = new List().AsReadOnly(); this.AvailableWriteEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); - this.AvailableReadEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + this.AvailableReadEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + this.AvailableWriteLocationByEndpoint = new ReadOnlyDictionary(new Dictionary()); + this.AvailableReadLocationByEndpoint = new ReadOnlyDictionary(new Dictionary()); this.WriteEndpoints = new List() { defaultEndpoint }.AsReadOnly(); this.AccountReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); - this.ReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); + this.ReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); + this.EffectivePreferredLocations = new List().AsReadOnly(); } public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) @@ -807,20 +903,27 @@ public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) this.AvailableWriteLocations = other.AvailableWriteLocations; this.AvailableReadLocations = other.AvailableReadLocations; this.AvailableWriteEndpointByLocation = other.AvailableWriteEndpointByLocation; - this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation; + this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation; + this.AvailableReadLocationByEndpoint = other.AvailableReadLocationByEndpoint; + this.AvailableWriteLocationByEndpoint = other.AvailableWriteLocationByEndpoint; this.WriteEndpoints = other.WriteEndpoints; this.AccountReadEndpoints = other.AccountReadEndpoints; - this.ReadEndpoints = other.ReadEndpoints; + this.ReadEndpoints = other.ReadEndpoints; + this.EffectivePreferredLocations = other.EffectivePreferredLocations; } public ReadOnlyCollection PreferredLocations { get; set; } public ReadOnlyCollection AvailableWriteLocations { get; set; } public ReadOnlyCollection AvailableReadLocations { get; set; } public ReadOnlyDictionary AvailableWriteEndpointByLocation { get; set; } - public ReadOnlyDictionary AvailableReadEndpointByLocation { get; set; } + public ReadOnlyDictionary AvailableReadEndpointByLocation { get; set; } + public ReadOnlyDictionary AvailableWriteLocationByEndpoint { get; set; } + public ReadOnlyDictionary AvailableReadLocationByEndpoint { get; set; } + public ReadOnlyCollection WriteEndpoints { get; set; } public ReadOnlyCollection ReadEndpoints { get; set; } public ReadOnlyCollection AccountReadEndpoints { get; set; } + public ReadOnlyCollection EffectivePreferredLocations { get; set; } } [Flags] From 7aeaa202d795101a6f2652e0f71504af40c04cb3 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 9 Sep 2024 19:09:37 -0400 Subject: [PATCH 03/16] LocationCacheTests changes. --- .../src/Routing/LocationCache.cs | 15 ++- .../LocationCacheTests.cs | 97 ++++++++++++++----- 2 files changed, 87 insertions(+), 25 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index 6734d68451..8f8061a048 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -124,6 +124,14 @@ public ReadOnlyCollection WriteEndpoints return this.locationInfo.WriteEndpoints; } + } + + public ReadOnlyCollection EffectivePreferredLocations + { + get + { + return this.locationInfo.EffectivePreferredLocations; + } } /// @@ -479,7 +487,12 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); - + + if (currentLocationInfo.PreferredLocations == null || currentLocationInfo.PreferredLocations.Count == 0) + { + mostPreferredLocation = currentLocationInfo.EffectivePreferredLocations.FirstOrDefault(); + } + // we should schedule refresh in background if we are unable to target the user's most preferredLocation. if (this.enableEndpointDiscovery) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs index f5070672d5..b731c0777a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs @@ -974,13 +974,15 @@ private static AccountProperties CreateDatabaseAccount( { { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, - { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, } : new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, }, WriteLocationsInternal = writeLocations @@ -1055,10 +1057,14 @@ private async Task ValidateLocationCacheAsync( bool useMultipleWriteLocations, bool endpointDiscoveryEnabled, bool isPreferredListEmpty) - { - for (int writeLocationIndex = 0; writeLocationIndex < 3; writeLocationIndex++) + { + + int maxWriteLocationIndex = 3; + int maxReadLocationIndex = isPreferredListEmpty ? 3 : 4; + + for (int writeLocationIndex = 0; writeLocationIndex < maxWriteLocationIndex; writeLocationIndex++) { - for (int readLocationIndex = 0; readLocationIndex < 2; readLocationIndex++) + for (int readLocationIndex = 0; readLocationIndex < maxReadLocationIndex; readLocationIndex++) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations, @@ -1087,21 +1093,53 @@ private async Task ValidateLocationCacheAsync( Dictionary readEndpointByLocation = this.databaseAccount.ReadableRegions.ToDictionary( location => location.Name, - location => new Uri(location.Endpoint)); - - Uri[] preferredAvailableWriteEndpoints = this.preferredLocations.Skip(writeLocationIndex) - .Where(location => writeEndpointByLocation.ContainsKey(location)) - .Select(location => writeEndpointByLocation[location]).ToArray(); - - Uri[] preferredAvailableReadEndpoints = this.preferredLocations.Skip(readLocationIndex) - .Where(location => readEndpointByLocation.ContainsKey(location)) - .Select(location => readEndpointByLocation[location]).ToArray(); + location => new Uri(location.Endpoint)); + + List accountLevelReadEndpoints = this.databaseAccount.ReadableRegions + .Where(accountRegion => readEndpointByLocation.ContainsKey(accountRegion.Name)) + .Select(accountRegion => readEndpointByLocation[accountRegion.Name]) + .ToList(); + + List accountLevelWriteEndpoints = this.databaseAccount.WritableRegions + .Where(accountRegion => writeEndpointByLocation.ContainsKey(accountRegion.Name)) + .Select(accountRegion => writeEndpointByLocation[accountRegion.Name]) + .ToList(); + + ReadOnlyCollection preferredLocationsWhenClientLevelPreferredLocationsIsEmpty = this.cache.EffectivePreferredLocations; + + Uri[] preferredAvailableWriteEndpoints, preferredAvailableReadEndpoints; + + if (isPreferredListEmpty) + { + preferredAvailableWriteEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(writeLocationIndex) + .Where(location => writeEndpointByLocation.ContainsKey(location)) + .Select(location => writeEndpointByLocation[location]).ToArray(); + + preferredAvailableReadEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(readLocationIndex) + .Where(location => readEndpointByLocation.ContainsKey(location)) + .Select(location => readEndpointByLocation[location]).ToArray(); + } + else + { + preferredAvailableWriteEndpoints = this.preferredLocations.Skip(writeLocationIndex) + .Where(location => writeEndpointByLocation.ContainsKey(location)) + .Select(location => writeEndpointByLocation[location]).ToArray(); + + preferredAvailableReadEndpoints = this.preferredLocations.Skip(readLocationIndex) + .Where(location => readEndpointByLocation.ContainsKey(location)) + .Select(location => readEndpointByLocation[location]).ToArray(); + } this.ValidateEndpointRefresh( useMultipleWriteLocations, - endpointDiscoveryEnabled, + endpointDiscoveryEnabled, + isPreferredListEmpty, preferredAvailableWriteEndpoints, - preferredAvailableReadEndpoints, + preferredAvailableReadEndpoints, + preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, + preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, + accountLevelWriteEndpoints, + accountLevelReadEndpoints, writeLocationIndex > 0, readLocationIndex > 0 && currentReadEndpoints[0] != LocationCacheTests.DefaultEndpoint, @@ -1150,9 +1188,14 @@ private async Task ValidateLocationCacheAsync( private void ValidateEndpointRefresh( bool useMultipleWriteLocations, - bool endpointDiscoveryEnabled, + bool endpointDiscoveryEnabled, + bool isPreferredListEmpty, Uri[] preferredAvailableWriteEndpoints, - Uri[] preferredAvailableReadEndpoints, + Uri[] preferredAvailableReadEndpoints, + ReadOnlyCollection preferredAvailableWriteRegions, + ReadOnlyCollection preferredAvailableReadRegions, + List accountLevelWriteEndpoints, + List accountLevelReadEndpoints, bool isFirstWriteEndpointUnavailable, bool isFirstReadEndpointUnavailable, bool hasMoreThanOneWriteEndpoints, @@ -1163,18 +1206,28 @@ private void ValidateEndpointRefresh( bool isMostPreferredLocationUnavailableForRead = isFirstReadEndpointUnavailable; bool isMostPreferredLocationUnavailableForWrite = useMultipleWriteLocations ? false : isFirstWriteEndpointUnavailable; - if (this.preferredLocations.Count > 0) + if (this.preferredLocations.Count > 0 || isPreferredListEmpty) { - string mostPreferredReadLocationName = this.preferredLocations.First(location => databaseAccount.ReadableRegions.Any(readLocation => readLocation.Name == location)); + string mostPreferredReadLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableReadRegions[0] : this.preferredLocations.FirstOrDefault(location => databaseAccount.ReadableRegions.Any(readLocation => readLocation.Name == location), ""); Uri mostPreferredReadEndpoint = LocationCacheTests.EndpointByLocation[mostPreferredReadLocationName]; isMostPreferredLocationUnavailableForRead = preferredAvailableReadEndpoints.Length == 0 ? true : (preferredAvailableReadEndpoints[0] != mostPreferredReadEndpoint); + + if (isPreferredListEmpty && endpointDiscoveryEnabled) + { + isMostPreferredLocationUnavailableForRead = preferredAvailableReadEndpoints[0] != accountLevelReadEndpoints[0]; + } - string mostPreferredWriteLocationName = this.preferredLocations.First(location => databaseAccount.WritableRegions.Any(writeLocation => writeLocation.Name == location)); + string mostPreferredWriteLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableWriteRegions[0] : this.preferredLocations.FirstOrDefault(location => databaseAccount.WritableRegions.Any(writeLocation => writeLocation.Name == location), ""); Uri mostPreferredWriteEndpoint = LocationCacheTests.EndpointByLocation[mostPreferredWriteLocationName]; if (useMultipleWriteLocations) { isMostPreferredLocationUnavailableForWrite = preferredAvailableWriteEndpoints.Length == 0 ? true : (preferredAvailableWriteEndpoints[0] != mostPreferredWriteEndpoint); + } + + if (isPreferredListEmpty && endpointDiscoveryEnabled) + { + isMostPreferredLocationUnavailableForWrite = preferredAvailableWriteEndpoints[0] != accountLevelWriteEndpoints[0]; } } @@ -1262,10 +1315,6 @@ private void ValidateRequestEndpointResolution( { firstAvailableReadEndpoint = LocationCacheTests.DefaultEndpoint; } - else if (this.preferredLocations.Count == 0) - { - firstAvailableReadEndpoint = firstAvailableWriteEndpoint; - } else if (availableReadEndpoints.Length > 0) { firstAvailableReadEndpoint = availableReadEndpoints[0]; From 5a6c044574d428d142a3473e488bfd22c5196bc9 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 11 Sep 2024 19:36:56 -0400 Subject: [PATCH 04/16] Update effective preferred regions count in GlobalEndpointManager.cs --- .../src/Routing/GlobalEndpointManager.cs | 289 ++++++++----- .../src/Routing/LocationCache.cs | 402 +++++++++--------- .../LocationCacheTests.cs | 152 +++---- 3 files changed, 448 insertions(+), 395 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 4e5184177b..9cc27cb405 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -38,6 +38,7 @@ internal class GlobalEndpointManager : IGlobalEndpointManager private readonly int backgroundRefreshLocationTimeIntervalInMS = GlobalEndpointManager.DefaultBackgroundRefreshLocationTimeIntervalInMS; private readonly object backgroundAccountRefreshLock = new object(); private readonly object isAccountRefreshInProgressLock = new object(); + private readonly ReaderWriterLock locationCacheReadWriteLock = new ReaderWriterLock(); private bool isAccountRefreshInProgress = false; private bool isBackgroundAccountRefreshActive = false; private DateTime LastBackgroundRefreshUtc = DateTime.MinValue; @@ -89,13 +90,21 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con } } - public ReadOnlyCollection ReadEndpoints => this.locationCache.ReadEndpoints; - + public ReadOnlyCollection ReadEndpoints => this.locationCache.ReadEndpoints; + public ReadOnlyCollection AccountReadEndpoints => this.locationCache.AccountReadEndpoints; - public ReadOnlyCollection WriteEndpoints => this.locationCache.WriteEndpoints; + public ReadOnlyCollection WriteEndpoints => this.locationCache.WriteEndpoints; - public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0; + public int PreferredLocationCount + { + get + { + Collection effectivePreferredLocations = this.GetEffectivePreferredLocations(); + + return effectivePreferredLocations.Count; + } + } public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request) { @@ -104,8 +113,8 @@ public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request) public Uri GetHubUri() { - return this.locationCache.GetHubUri(); - } + return this.locationCache.GetHubUri(); + } /// /// This will get the account information. @@ -116,19 +125,19 @@ public Uri GetHubUri() /// public static async Task GetDatabaseAccountFromAnyLocationsAsync( Uri defaultEndpoint, - IList? locations, + IList? locations, IList? accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) - { + { using (GetAccountPropertiesHelper threadSafeGetAccountHelper = new GetAccountPropertiesHelper( defaultEndpoint, - locations, + locations, accountInitializationCustomEndpoints, getDatabaseAccountFn, - cancellationToken)) - { - return await threadSafeGetAccountHelper.GetAccountPropertiesAsync(); + cancellationToken)) + { + return await threadSafeGetAccountHelper.GetAccountPropertiesAsync(); } } @@ -139,30 +148,30 @@ private class GetAccountPropertiesHelper : IDisposable { private readonly CancellationTokenSource CancellationTokenSource; private readonly Uri DefaultEndpoint; - private readonly bool LimitToGlobalEndpointOnly; + private readonly bool LimitToGlobalEndpointOnly; private readonly IEnumerator ServiceEndpointEnumerator; private readonly Func> GetDatabaseAccountFn; private readonly List TransientExceptions = new List(); private AccountProperties? AccountProperties = null; - private Exception? NonRetriableException = null; + private Exception? NonRetriableException = null; private int disposeCounter = 0; public GetAccountPropertiesHelper( Uri defaultEndpoint, - IList? locations, + IList? locations, IList? accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { this.DefaultEndpoint = defaultEndpoint; this.LimitToGlobalEndpointOnly = (locations == null || locations.Count == 0) && (accountInitializationCustomEndpoints == null || accountInitializationCustomEndpoints.Count == 0); - this.GetDatabaseAccountFn = getDatabaseAccountFn; - this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - this.ServiceEndpointEnumerator = GetAccountPropertiesHelper - .GetServiceEndpoints( - defaultEndpoint, - locations, - accountInitializationCustomEndpoints) + this.GetDatabaseAccountFn = getDatabaseAccountFn; + this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + this.ServiceEndpointEnumerator = GetAccountPropertiesHelper + .GetServiceEndpoints( + defaultEndpoint, + locations, + accountInitializationCustomEndpoints) .GetEnumerator(); } @@ -173,7 +182,7 @@ public async Task GetAccountPropertiesAsync() { return await this.GetOnlyGlobalEndpointAsync(); } - + Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); // Start a timer to start secondary requests in parallel. @@ -263,7 +272,7 @@ private async Task GetOnlyGlobalEndpointAsync() /// This is done in a thread safe way to allow multiple tasks to iterate over the list of service endpoints. /// private async Task TryGetAccountPropertiesFromAllLocationsAsync() - { + { while (this.TryMoveNextServiceEndpointhreadSafe( out Uri? serviceEndpoint)) { @@ -273,19 +282,19 @@ private async Task TryGetAccountPropertiesFromAllLocationsAsync() return; } - await this.GetAndUpdateAccountPropertiesAsync( + await this.GetAndUpdateAccountPropertiesAsync( endpoint: serviceEndpoint); } - } - - /// - /// We first iterate through all the private endpoints to fetch the account information. - /// If all the attempt fails to fetch the metadata from the private endpoints, we will - /// attempt to retrieve the account information from the regional endpoints constructed - /// using the preferred regions list. - /// - /// An instance of that will contain the service endpoint. - /// A boolean flag indicating if the was advanced in a thread safe manner. + } + + /// + /// We first iterate through all the private endpoints to fetch the account information. + /// If all the attempt fails to fetch the metadata from the private endpoints, we will + /// attempt to retrieve the account information from the regional endpoints constructed + /// using the preferred regions list. + /// + /// An instance of that will contain the service endpoint. + /// A boolean flag indicating if the was advanced in a thread safe manner. private bool TryMoveNextServiceEndpointhreadSafe( out Uri? serviceEndpoint) { @@ -306,7 +315,7 @@ private bool TryMoveNextServiceEndpointhreadSafe( serviceEndpoint = this.ServiceEndpointEnumerator.Current; return true; } - } + } private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) { @@ -358,81 +367,81 @@ private static bool IsNonRetriableException(Exception exception) } return false; - } - - /// - /// Returns an instance of containing the private and regional service endpoints to iterate over. - /// - /// An instance of containing the default global endpoint. - /// An instance of containing the preferred serviceEndpoint names. - /// An instance of containing the custom private endpoints. - /// An instance of containing the service endpoints. - private static IEnumerable GetServiceEndpoints( - Uri defaultEndpoint, - IList? locations, - IList? accountInitializationCustomEndpoints) - { - // We first iterate over all the private endpoints and yield return them. - if (accountInitializationCustomEndpoints?.Count > 0) - { - foreach (Uri customEndpoint in accountInitializationCustomEndpoints) - { - // Yield return all of the custom private endpoints first. - yield return customEndpoint; - } - } - - // The next step is to iterate over the preferred locations, construct and yield return the regional endpoints one by one. - // The regional endpoints will be constructed by appending the preferred region name as a suffix to the default global endpoint. - if (locations?.Count > 0) - { - foreach (string location in locations) - { - // Yield return all of the regional endpoints once the private custom endpoints are visited. - yield return LocationHelper.GetLocationEndpoint(defaultEndpoint, location); - } - } - } - - public void Dispose() - { - if (Interlocked.Increment(ref this.disposeCounter) == 1) - { - this.CancellationTokenSource?.Cancel(); - this.CancellationTokenSource?.Dispose(); - } - } + } + + /// + /// Returns an instance of containing the private and regional service endpoints to iterate over. + /// + /// An instance of containing the default global endpoint. + /// An instance of containing the preferred serviceEndpoint names. + /// An instance of containing the custom private endpoints. + /// An instance of containing the service endpoints. + private static IEnumerable GetServiceEndpoints( + Uri defaultEndpoint, + IList? locations, + IList? accountInitializationCustomEndpoints) + { + // We first iterate over all the private endpoints and yield return them. + if (accountInitializationCustomEndpoints?.Count > 0) + { + foreach (Uri customEndpoint in accountInitializationCustomEndpoints) + { + // Yield return all of the custom private endpoints first. + yield return customEndpoint; + } + } + + // The next step is to iterate over the preferred locations, construct and yield return the regional endpoints one by one. + // The regional endpoints will be constructed by appending the preferred region name as a suffix to the default global endpoint. + if (locations?.Count > 0) + { + foreach (string location in locations) + { + // Yield return all of the regional endpoints once the private custom endpoints are visited. + yield return LocationHelper.GetLocationEndpoint(defaultEndpoint, location); + } + } + } + + public void Dispose() + { + if (Interlocked.Increment(ref this.disposeCounter) == 1) + { + this.CancellationTokenSource?.Cancel(); + this.CancellationTokenSource?.Dispose(); + } + } } public virtual Uri ResolveServiceEndpoint(DocumentServiceRequest request) { return this.locationCache.ResolveServiceEndpoint(request); - } - - /// - /// Gets the default endpoint of the account - /// - /// the default endpoint. - public Uri GetDefaultEndpoint() - { - return this.locationCache.GetDefaultEndpoint(); - } - - /// - /// Gets the mapping of available write region names to the respective endpoints - /// - public ReadOnlyDictionary GetAvailableWriteEndpointsByLocation() - { - return this.locationCache.GetAvailableWriteEndpointsByLocation(); - } - - /// - /// Gets the mapping of available read region names to the respective endpoints - /// - public ReadOnlyDictionary GetAvailableReadEndpointsByLocation() - { - return this.locationCache.GetAvailableReadEndpointsByLocation(); - } + } + + /// + /// Gets the default endpoint of the account + /// + /// the default endpoint. + public Uri GetDefaultEndpoint() + { + return this.locationCache.GetDefaultEndpoint(); + } + + /// + /// Gets the mapping of available write region names to the respective endpoints + /// + public ReadOnlyDictionary GetAvailableWriteEndpointsByLocation() + { + return this.locationCache.GetAvailableWriteEndpointsByLocation(); + } + + /// + /// Gets the mapping of available read region names to the respective endpoints + /// + public ReadOnlyDictionary GetAvailableReadEndpointsByLocation() + { + return this.locationCache.GetAvailableReadEndpointsByLocation(); + } /// /// Returns serviceEndpoint corresponding to the endpoint @@ -447,11 +456,11 @@ public ReadOnlyCollection GetApplicableEndpoints(DocumentServiceRequest req { return this.locationCache.GetApplicableEndpoints(request, isReadRequest); } - + public ReadOnlyCollection GetApplicableRegions(IEnumerable excludeRegions, bool isReadRequest) { return this.locationCache.GetApplicableRegions(excludeRegions, isReadRequest); - } + } public bool TryGetLocationForGatewayDiagnostics(Uri endpoint, out string regionName) { @@ -497,8 +506,16 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account return; } - this.locationCache.OnDatabaseAccountRead(databaseAccount); - + this.locationCacheReadWriteLock.AcquireWriterLock(TimeSpan.FromMilliseconds(10)); + try + { + this.locationCache.OnDatabaseAccountRead(databaseAccount); + } + finally + { + this.locationCacheReadWriteLock.ReleaseWriterLock(); + } + if (this.isBackgroundAccountRefreshActive) { return; @@ -633,13 +650,25 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh) try { this.LastBackgroundRefreshUtc = DateTime.UtcNow; - this.locationCache.OnDatabaseAccountRead(await this.GetDatabaseAccountAsync(true)); - } - catch (Exception ex) - { - DefaultTrace.TraceWarning("Failed to refresh database account with exception: {0}. Activity Id: '{1}'", - ex, - System.Diagnostics.Trace.CorrelationManager.ActivityId); + + AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true); + + this.locationCacheReadWriteLock.AcquireWriterLock(TimeSpan.FromMilliseconds(10)); + + try + { + this.locationCache.OnDatabaseAccountRead(accountProperties); + } + finally + { + this.locationCacheReadWriteLock.ReleaseWriterLock(); + } + } + catch (Exception ex) + { + DefaultTrace.TraceWarning("Failed to refresh database account with exception: {0}. Activity Id: '{1}'", + ex, + System.Diagnostics.Trace.CorrelationManager.ActivityId); } finally { @@ -657,7 +686,7 @@ internal async Task GetDatabaseAccountAsync(bool forceRefresh obsoleteValue: null, singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( this.defaultEndpoint, - this.connectionPolicy.PreferredLocations, + this.connectionPolicy.PreferredLocations, this.connectionPolicy.AccountInitializationCustomEndpoints, this.GetDatabaseAccountAsync, this.cancellationTokenSource.Token), @@ -676,5 +705,29 @@ private bool SkipRefresh(bool forceRefresh) return (this.isAccountRefreshInProgress || this.MinTimeBetweenAccountRefresh > timeSinceLastRefresh) && !forceRefresh; } + + private Collection GetEffectivePreferredLocations() + { + if (this.connectionPolicy.PreferredLocations != null && this.connectionPolicy.PreferredLocations.Count > 0) + { + return this.connectionPolicy.PreferredLocations; + } + + this.locationCacheReadWriteLock.AcquireReaderLock(TimeSpan.FromMilliseconds(10)); + + try + { + if (this.databaseAccountCache.Keys == null || this.databaseAccountCache.Keys.Count == 0) + { + return new Collection(); + } + + return new Collection(this.locationCache.EffectivePreferredLocations); + } + finally + { + this.locationCacheReadWriteLock.ReleaseReaderLock(); + } + } } } diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index 8f8061a048..604de0b157 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -8,9 +8,9 @@ namespace Microsoft.Azure.Cosmos.Routing using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; - using System.Linq; - using Microsoft.Azure.Cosmos.Core.Trace; - using Microsoft.Azure.Cosmos.Linq; + using System.Linq; + using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Cosmos.Linq; using Microsoft.Azure.Documents; /// @@ -124,16 +124,10 @@ public ReadOnlyCollection WriteEndpoints return this.locationInfo.WriteEndpoints; } - } - - public ReadOnlyCollection EffectivePreferredLocations - { - get - { - return this.locationInfo.EffectivePreferredLocations; - } } + public ReadOnlyCollection EffectivePreferredLocations => this.locationInfo.EffectivePreferredLocations; + /// /// Returns the location corresponding to the endpoint if location specific endpoint is provided. /// For the defaultEndPoint, we will return the first available write location. @@ -266,11 +260,11 @@ public Uri GetHubUri() string writeLocation = currentLocationInfo.AvailableWriteLocations[0]; Uri locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation]; return locationEndpointToRoute; - } - - public ReadOnlyCollection GetAvailableReadLocations() - { - return this.locationInfo.AvailableReadLocations; + } + + public ReadOnlyCollection GetAvailableReadLocations() + { + return this.locationInfo.AvailableReadLocations; } /// @@ -332,145 +326,145 @@ public ReadOnlyCollection GetApplicableEndpoints(DocumentServiceRequest req if (request.RequestContext.ExcludeRegions == null || request.RequestContext.ExcludeRegions.Count == 0) { return isReadRequest ? this.ReadEndpoints : this.WriteEndpoints; - } - - DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; - - ReadOnlyCollection effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.PreferredLocations; - - if (effectivePreferredLocations == null || effectivePreferredLocations.Count == 0) - { - effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations; + } + + DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; + + ReadOnlyCollection effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.PreferredLocations; + + if (effectivePreferredLocations == null || effectivePreferredLocations.Count == 0) + { + effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations; } return this.GetApplicableEndpoints( isReadRequest ? this.locationInfo.AvailableReadEndpointByLocation : this.locationInfo.AvailableWriteEndpointByLocation, - effectivePreferredLocations, + effectivePreferredLocations, this.defaultEndpoint, request.RequestContext.ExcludeRegions); - } - + } + public ReadOnlyCollection GetApplicableRegions(IEnumerable excludeRegions, bool isReadRequest) - { - bool isPreferredLocationsEmpty = this.locationInfo.PreferredLocations == null || this.locationInfo.PreferredLocations.Count == 0; - - DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; - - ReadOnlyCollection effectivePreferredLocations = this.locationInfo.PreferredLocations; - - if (effectivePreferredLocations == null || effectivePreferredLocations.Count == 0) - { - effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations; - } - - Uri firstEffectivePreferredEndpoint = isReadRequest ? databaseAccountLocationsInfoSnapshot.ReadEndpoints[0] : databaseAccountLocationsInfoSnapshot.WriteEndpoints[0]; - - if (isReadRequest) - { - databaseAccountLocationsInfoSnapshot.AvailableReadLocationByEndpoint.TryGetValue(firstEffectivePreferredEndpoint, out string firstEffectivePreferredLocation); - - return this.GetApplicableRegions( - databaseAccountLocationsInfoSnapshot.AvailableReadLocations, - effectivePreferredLocations, - isPreferredLocationsEmpty ? firstEffectivePreferredLocation : databaseAccountLocationsInfoSnapshot.PreferredLocations[0], - excludeRegions); - } - else - { - databaseAccountLocationsInfoSnapshot.AvailableWriteLocationByEndpoint.TryGetValue(firstEffectivePreferredEndpoint, out string firstEffectivePreferredLocation); - - return this.GetApplicableRegions( - databaseAccountLocationsInfoSnapshot.AvailableWriteLocations, - effectivePreferredLocations, - isPreferredLocationsEmpty ? firstEffectivePreferredLocation : databaseAccountLocationsInfoSnapshot.PreferredLocations[0], - excludeRegions); + { + bool isPreferredLocationsEmpty = this.locationInfo.PreferredLocations == null || this.locationInfo.PreferredLocations.Count == 0; + + DatabaseAccountLocationsInfo databaseAccountLocationsInfoSnapshot = this.locationInfo; + + ReadOnlyCollection effectivePreferredLocations = this.locationInfo.PreferredLocations; + + if (effectivePreferredLocations == null || effectivePreferredLocations.Count == 0) + { + effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations; } - } - + + Uri firstEffectivePreferredEndpoint = isReadRequest ? databaseAccountLocationsInfoSnapshot.ReadEndpoints[0] : databaseAccountLocationsInfoSnapshot.WriteEndpoints[0]; + + if (isReadRequest) + { + databaseAccountLocationsInfoSnapshot.AvailableReadLocationByEndpoint.TryGetValue(firstEffectivePreferredEndpoint, out string firstEffectivePreferredLocation); + + return this.GetApplicableRegions( + databaseAccountLocationsInfoSnapshot.AvailableReadLocations, + effectivePreferredLocations, + isPreferredLocationsEmpty ? firstEffectivePreferredLocation : databaseAccountLocationsInfoSnapshot.PreferredLocations[0], + excludeRegions); + } + else + { + databaseAccountLocationsInfoSnapshot.AvailableWriteLocationByEndpoint.TryGetValue(firstEffectivePreferredEndpoint, out string firstEffectivePreferredLocation); + + return this.GetApplicableRegions( + databaseAccountLocationsInfoSnapshot.AvailableWriteLocations, + effectivePreferredLocations, + isPreferredLocationsEmpty ? firstEffectivePreferredLocation : databaseAccountLocationsInfoSnapshot.PreferredLocations[0], + excludeRegions); + } + } + /// /// Gets applicable endpoints for a request, if there are no applicable endpoints, returns the fallback endpoint /// - /// + /// /// /// /// /// a list of applicable endpoints for a request private ReadOnlyCollection GetApplicableEndpoints( - ReadOnlyDictionary regionNameByEndpoint, + ReadOnlyDictionary regionNameByEndpoint, ReadOnlyCollection effectivePreferredLocations, Uri fallbackEndpoint, IEnumerable excludeRegions) - { - List applicableEndpoints = new List(regionNameByEndpoint.Count); - HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); - - if (excludeRegions != null) - { - foreach (string region in effectivePreferredLocations) - { - if (!excludeRegionsHash.Contains(region) - && regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) - { - applicableEndpoints.Add(endpoint); - } - } - } - else - { - foreach (string region in effectivePreferredLocations) - { - if (regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) - { - applicableEndpoints.Add(endpoint); - } - } - } - - if (applicableEndpoints.Count == 0) - { - applicableEndpoints.Add(fallbackEndpoint); - } - - return new ReadOnlyCollection(applicableEndpoints); - } - + { + List applicableEndpoints = new List(regionNameByEndpoint.Count); + HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); + + if (excludeRegions != null) + { + foreach (string region in effectivePreferredLocations) + { + if (!excludeRegionsHash.Contains(region) + && regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) + { + applicableEndpoints.Add(endpoint); + } + } + } + else + { + foreach (string region in effectivePreferredLocations) + { + if (regionNameByEndpoint.TryGetValue(region, out Uri endpoint)) + { + applicableEndpoints.Add(endpoint); + } + } + } + + if (applicableEndpoints.Count == 0) + { + applicableEndpoints.Add(fallbackEndpoint); + } + + return new ReadOnlyCollection(applicableEndpoints); + } + /// /// Gets applicable endpoints for a request, if there are no applicable endpoints, returns the fallback endpoint /// - /// + /// /// /// /// /// a list of applicable endpoints for a request private ReadOnlyCollection GetApplicableRegions( - ReadOnlyCollection availableLocations, + ReadOnlyCollection availableLocations, ReadOnlyCollection effectivePreferredLocations, string fallbackRegion, IEnumerable excludeRegions) { - List applicableRegions = new List(availableLocations.Count); - HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); - - if (excludeRegions != null) - { - foreach (string region in effectivePreferredLocations) - { - if (availableLocations.Contains(region) - && !excludeRegionsHash.Contains(region)) - { - applicableRegions.Add(region); - } - } - } - else - { - foreach (string region in effectivePreferredLocations) - { - if (availableLocations.Contains(region)) - { - applicableRegions.Add(region); - } - } + List applicableRegions = new List(availableLocations.Count); + HashSet excludeRegionsHash = excludeRegions == null ? null : new HashSet(excludeRegions); + + if (excludeRegions != null) + { + foreach (string region in effectivePreferredLocations) + { + if (availableLocations.Contains(region) + && !excludeRegionsHash.Contains(region)) + { + applicableRegions.Add(region); + } + } + } + else + { + foreach (string region in effectivePreferredLocations) + { + if (availableLocations.Contains(region)) + { + applicableRegions.Add(region); + } + } } if (applicableRegions.Count == 0) @@ -486,12 +480,12 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) canRefreshInBackground = true; DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; - string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); - - if (currentLocationInfo.PreferredLocations == null || currentLocationInfo.PreferredLocations.Count == 0) - { - mostPreferredLocation = currentLocationInfo.EffectivePreferredLocations.FirstOrDefault(); - } + string mostPreferredLocation = currentLocationInfo.PreferredLocations.FirstOrDefault(); + + if (currentLocationInfo.PreferredLocations == null || currentLocationInfo.PreferredLocations.Count == 0) + { + mostPreferredLocation = currentLocationInfo.EffectivePreferredLocations.FirstOrDefault(); + } // we should schedule refresh in background if we are unable to target the user's most preferredLocation. if (this.enableEndpointDiscovery) @@ -523,7 +517,7 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) // other available read endpoints DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not available for read.", mostPreferredLocation); return true; - } + } } else { @@ -559,7 +553,7 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) if (currentLocationInfo.AvailableWriteEndpointByLocation.TryGetValue(mostPreferredLocation, out mostPreferredWriteEndpoint)) { shouldRefresh |= mostPreferredWriteEndpoint != writeLocationEndpoints[0]; - DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation); + DefaultTrace.TraceInformation("ShouldRefreshEndpoints = {0} since most preferred location {1} is not available for write.", shouldRefresh, mostPreferredLocation); return shouldRefresh; } else @@ -724,23 +718,23 @@ private void UpdateLocationCache( endpointsByLocation: nextLocationInfo.AvailableReadEndpointByLocation, orderedLocations: nextLocationInfo.AvailableReadLocations, expectedAvailableOperation: OperationType.Read, - fallbackEndpoint: nextLocationInfo.WriteEndpoints[0]); - - if (preferenceList == null || preferenceList.Count == 0) - { - if (!nextLocationInfo.AvailableReadLocationByEndpoint.TryGetValue(this.defaultEndpoint, out string regionForDefaultEndpoint)) - { - nextLocationInfo.EffectivePreferredLocations = nextLocationInfo.AvailableReadLocations; - } - else - { - List locations = new () - { - regionForDefaultEndpoint - }; - - nextLocationInfo.EffectivePreferredLocations = new ReadOnlyCollection(locations); - } + fallbackEndpoint: nextLocationInfo.WriteEndpoints[0]); + + if (nextLocationInfo.PreferredLocations == null || nextLocationInfo.PreferredLocations.Count == 0) + { + if (!nextLocationInfo.AvailableReadLocationByEndpoint.TryGetValue(this.defaultEndpoint, out string regionForDefaultEndpoint)) + { + nextLocationInfo.EffectivePreferredLocations = nextLocationInfo.AvailableReadLocations; + } + else + { + List locations = new () + { + regionForDefaultEndpoint + }; + + nextLocationInfo.EffectivePreferredLocations = new ReadOnlyCollection(locations); + } } this.lastCacheUpdateTimestamp = DateTime.UtcNow; @@ -770,45 +764,45 @@ private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionar // If client can use multiple write locations, preferred locations list should be used for determining // both read and write endpoints order. - if (currentLocationInfo.PreferredLocations != null && currentLocationInfo.PreferredLocations.Count >= 1) - { - foreach (string location in currentLocationInfo.PreferredLocations) - { - if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) - { - if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) - { - unavailableEndpoints.Add(endpoint); - } - else - { - endpoints.Add(endpoint); - } - } - } - } - else - { - foreach (string location in orderedLocations) - { - if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) - { - if (this.defaultEndpoint.Equals(endpoint)) - { - endpoints = new List(); - break; - } - - if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) - { - unavailableEndpoints.Add(endpoint); - } - else - { - endpoints.Add(endpoint); - } - } - } + if (currentLocationInfo.PreferredLocations != null && currentLocationInfo.PreferredLocations.Count >= 1) + { + foreach (string location in currentLocationInfo.PreferredLocations) + { + if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) + { + if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + { + unavailableEndpoints.Add(endpoint); + } + else + { + endpoints.Add(endpoint); + } + } + } + } + else + { + foreach (string location in orderedLocations) + { + if (endpointsByLocation.TryGetValue(location, out Uri endpoint)) + { + if (this.defaultEndpoint.Equals(endpoint)) + { + endpoints = new List(); + break; + } + + if (this.IsEndpointUnavailable(endpoint, expectedAvailableOperation)) + { + unavailableEndpoints.Add(endpoint); + } + else + { + endpoints.Add(endpoint); + } + } + } } if (endpoints.Count == 0) @@ -842,9 +836,9 @@ private ReadOnlyCollection GetPreferredAvailableEndpoints(ReadOnlyDictionar private ReadOnlyDictionary GetEndpointByLocation(IEnumerable locations, out ReadOnlyCollection orderedLocations, out ReadOnlyDictionary availableLocationsByEndpoint) { - Dictionary endpointsByLocation = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary endpointsByLocation = new Dictionary(StringComparer.OrdinalIgnoreCase); Dictionary mutableAvailableLocationsByEndpoint = new Dictionary(); - + List parsedLocations = new List(); foreach (AccountRegion location in locations) @@ -854,9 +848,9 @@ private ReadOnlyDictionary GetEndpointByLocation(IEnumerable GetEndpointByLocation(IEnumerable(mutableAvailableLocationsByEndpoint); + orderedLocations = parsedLocations.AsReadOnly(); + availableLocationsByEndpoint = new ReadOnlyDictionary(mutableAvailableLocationsByEndpoint); return new ReadOnlyDictionary(endpointsByLocation); } @@ -901,12 +895,12 @@ public DatabaseAccountLocationsInfo(ReadOnlyCollection preferredLocation this.AvailableWriteLocations = new List().AsReadOnly(); this.AvailableReadLocations = new List().AsReadOnly(); this.AvailableWriteEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); - this.AvailableReadEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); - this.AvailableWriteLocationByEndpoint = new ReadOnlyDictionary(new Dictionary()); + this.AvailableReadEndpointByLocation = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + this.AvailableWriteLocationByEndpoint = new ReadOnlyDictionary(new Dictionary()); this.AvailableReadLocationByEndpoint = new ReadOnlyDictionary(new Dictionary()); this.WriteEndpoints = new List() { defaultEndpoint }.AsReadOnly(); this.AccountReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); - this.ReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); + this.ReadEndpoints = new List() { defaultEndpoint }.AsReadOnly(); this.EffectivePreferredLocations = new List().AsReadOnly(); } @@ -916,12 +910,12 @@ public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) this.AvailableWriteLocations = other.AvailableWriteLocations; this.AvailableReadLocations = other.AvailableReadLocations; this.AvailableWriteEndpointByLocation = other.AvailableWriteEndpointByLocation; - this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation; - this.AvailableReadLocationByEndpoint = other.AvailableReadLocationByEndpoint; + this.AvailableReadEndpointByLocation = other.AvailableReadEndpointByLocation; + this.AvailableReadLocationByEndpoint = other.AvailableReadLocationByEndpoint; this.AvailableWriteLocationByEndpoint = other.AvailableWriteLocationByEndpoint; this.WriteEndpoints = other.WriteEndpoints; this.AccountReadEndpoints = other.AccountReadEndpoints; - this.ReadEndpoints = other.ReadEndpoints; + this.ReadEndpoints = other.ReadEndpoints; this.EffectivePreferredLocations = other.EffectivePreferredLocations; } @@ -929,14 +923,14 @@ public DatabaseAccountLocationsInfo(DatabaseAccountLocationsInfo other) public ReadOnlyCollection AvailableWriteLocations { get; set; } public ReadOnlyCollection AvailableReadLocations { get; set; } public ReadOnlyDictionary AvailableWriteEndpointByLocation { get; set; } - public ReadOnlyDictionary AvailableReadEndpointByLocation { get; set; } + public ReadOnlyDictionary AvailableReadEndpointByLocation { get; set; } public ReadOnlyDictionary AvailableWriteLocationByEndpoint { get; set; } - public ReadOnlyDictionary AvailableReadLocationByEndpoint { get; set; } + public ReadOnlyDictionary AvailableReadLocationByEndpoint { get; set; } public ReadOnlyCollection WriteEndpoints { get; set; } public ReadOnlyCollection ReadEndpoints { get; set; } public ReadOnlyCollection AccountReadEndpoints { get; set; } - public ReadOnlyCollection EffectivePreferredLocations { get; set; } + public ReadOnlyCollection EffectivePreferredLocations { get; set; } } [Flags] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs index b731c0777a..df3c92f7ed 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs @@ -174,15 +174,15 @@ await BackoffRetryUtility.ExecuteAsync( } private ClientRetryPolicy CreateClientRetryPolicy( - bool enableEndpointDiscovery, + bool enableEndpointDiscovery, bool partitionLevelFailoverEnabled, GlobalEndpointManager endpointManager) { return new ClientRetryPolicy( endpointManager, this.partitionKeyRangeLocationCache, - new RetryOptions(), - enableEndpointDiscovery, + new RetryOptions(), + enableEndpointDiscovery, isPertitionLevelFailoverEnabled: partitionLevelFailoverEnabled); } @@ -716,7 +716,7 @@ public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( preferedRegionListOverride: preferredList, enforceSingleMasterSingleWriteLocation: true); - endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); + endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: enablePartitionLevelFailover, endpointManager); @@ -974,7 +974,7 @@ private static AccountProperties CreateDatabaseAccount( { { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, - { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, } : @@ -982,7 +982,7 @@ private static AccountProperties CreateDatabaseAccount( { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, - { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, + { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, }, WriteLocationsInternal = writeLocations @@ -1057,10 +1057,14 @@ private async Task ValidateLocationCacheAsync( bool useMultipleWriteLocations, bool endpointDiscoveryEnabled, bool isPreferredListEmpty) - { - - int maxWriteLocationIndex = 3; - int maxReadLocationIndex = isPreferredListEmpty ? 3 : 4; + { + + // hardcoded to represent - (location1, location2, location3) as the write regions (with and without preferred regions set) + int maxWriteLocationIndex = 3; + + // hardcoded to represent - (location1, location2, location3, location4) as the account regions and (location1, location2, location3) + // as the read regions (with preferred regions set) + int maxReadLocationIndex = isPreferredListEmpty ? 4 : 3; for (int writeLocationIndex = 0; writeLocationIndex < maxWriteLocationIndex; writeLocationIndex++) { @@ -1091,54 +1095,54 @@ private async Task ValidateLocationCacheAsync( location => location.Name, location => new Uri(location.Endpoint)); - Dictionary readEndpointByLocation = this.databaseAccount.ReadableRegions.ToDictionary( + Dictionary readEndpointByLocation = this.databaseAccount.ReadLocationsInternal.ToDictionary( location => location.Name, - location => new Uri(location.Endpoint)); - - List accountLevelReadEndpoints = this.databaseAccount.ReadableRegions - .Where(accountRegion => readEndpointByLocation.ContainsKey(accountRegion.Name)) - .Select(accountRegion => readEndpointByLocation[accountRegion.Name]) - .ToList(); - - List accountLevelWriteEndpoints = this.databaseAccount.WritableRegions - .Where(accountRegion => writeEndpointByLocation.ContainsKey(accountRegion.Name)) - .Select(accountRegion => writeEndpointByLocation[accountRegion.Name]) - .ToList(); - - ReadOnlyCollection preferredLocationsWhenClientLevelPreferredLocationsIsEmpty = this.cache.EffectivePreferredLocations; - - Uri[] preferredAvailableWriteEndpoints, preferredAvailableReadEndpoints; - - if (isPreferredListEmpty) - { - preferredAvailableWriteEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(writeLocationIndex) - .Where(location => writeEndpointByLocation.ContainsKey(location)) - .Select(location => writeEndpointByLocation[location]).ToArray(); - - preferredAvailableReadEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(readLocationIndex) - .Where(location => readEndpointByLocation.ContainsKey(location)) - .Select(location => readEndpointByLocation[location]).ToArray(); - } - else - { - preferredAvailableWriteEndpoints = this.preferredLocations.Skip(writeLocationIndex) - .Where(location => writeEndpointByLocation.ContainsKey(location)) - .Select(location => writeEndpointByLocation[location]).ToArray(); - - preferredAvailableReadEndpoints = this.preferredLocations.Skip(readLocationIndex) - .Where(location => readEndpointByLocation.ContainsKey(location)) - .Select(location => readEndpointByLocation[location]).ToArray(); + location => new Uri(location.Endpoint)); + + List accountLevelReadEndpoints = this.databaseAccount.ReadLocationsInternal + .Where(accountRegion => readEndpointByLocation.ContainsKey(accountRegion.Name)) + .Select(accountRegion => readEndpointByLocation[accountRegion.Name]) + .ToList(); + + List accountLevelWriteEndpoints = this.databaseAccount.WriteLocationsInternal + .Where(accountRegion => writeEndpointByLocation.ContainsKey(accountRegion.Name)) + .Select(accountRegion => writeEndpointByLocation[accountRegion.Name]) + .ToList(); + + ReadOnlyCollection preferredLocationsWhenClientLevelPreferredLocationsIsEmpty = this.cache.EffectivePreferredLocations; + + Uri[] preferredAvailableWriteEndpoints, preferredAvailableReadEndpoints; + + if (isPreferredListEmpty) + { + preferredAvailableWriteEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(writeLocationIndex) + .Where(location => writeEndpointByLocation.ContainsKey(location)) + .Select(location => writeEndpointByLocation[location]).ToArray(); + + preferredAvailableReadEndpoints = preferredLocationsWhenClientLevelPreferredLocationsIsEmpty.Skip(readLocationIndex) + .Where(location => readEndpointByLocation.ContainsKey(location)) + .Select(location => readEndpointByLocation[location]).ToArray(); + } + else + { + preferredAvailableWriteEndpoints = this.preferredLocations.Skip(writeLocationIndex) + .Where(location => writeEndpointByLocation.ContainsKey(location)) + .Select(location => writeEndpointByLocation[location]).ToArray(); + + preferredAvailableReadEndpoints = this.preferredLocations.Skip(readLocationIndex) + .Where(location => readEndpointByLocation.ContainsKey(location)) + .Select(location => readEndpointByLocation[location]).ToArray(); } this.ValidateEndpointRefresh( useMultipleWriteLocations, - endpointDiscoveryEnabled, + endpointDiscoveryEnabled, isPreferredListEmpty, preferredAvailableWriteEndpoints, - preferredAvailableReadEndpoints, - preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, - preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, - accountLevelWriteEndpoints, + preferredAvailableReadEndpoints, + preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, + preferredLocationsWhenClientLevelPreferredLocationsIsEmpty, + accountLevelWriteEndpoints, accountLevelReadEndpoints, writeLocationIndex > 0, readLocationIndex > 0 && @@ -1188,13 +1192,13 @@ private async Task ValidateLocationCacheAsync( private void ValidateEndpointRefresh( bool useMultipleWriteLocations, - bool endpointDiscoveryEnabled, + bool endpointDiscoveryEnabled, bool isPreferredListEmpty, Uri[] preferredAvailableWriteEndpoints, - Uri[] preferredAvailableReadEndpoints, - ReadOnlyCollection preferredAvailableWriteRegions, - ReadOnlyCollection preferredAvailableReadRegions, - List accountLevelWriteEndpoints, + Uri[] preferredAvailableReadEndpoints, + ReadOnlyCollection preferredAvailableWriteRegions, + ReadOnlyCollection preferredAvailableReadRegions, + List accountLevelWriteEndpoints, List accountLevelReadEndpoints, bool isFirstWriteEndpointUnavailable, bool isFirstReadEndpointUnavailable, @@ -1206,28 +1210,29 @@ private void ValidateEndpointRefresh( bool isMostPreferredLocationUnavailableForRead = isFirstReadEndpointUnavailable; bool isMostPreferredLocationUnavailableForWrite = useMultipleWriteLocations ? false : isFirstWriteEndpointUnavailable; - if (this.preferredLocations.Count > 0 || isPreferredListEmpty) + + if (this.preferredLocations.Count > 0 || (isPreferredListEmpty && endpointDiscoveryEnabled)) { - string mostPreferredReadLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableReadRegions[0] : this.preferredLocations.FirstOrDefault(location => databaseAccount.ReadableRegions.Any(readLocation => readLocation.Name == location), ""); + string mostPreferredReadLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableReadRegions[0] : this.preferredLocations.FirstOrDefault(location => this.databaseAccount.ReadableRegions.Any(readLocation => readLocation.Name == location), ""); Uri mostPreferredReadEndpoint = LocationCacheTests.EndpointByLocation[mostPreferredReadLocationName]; isMostPreferredLocationUnavailableForRead = preferredAvailableReadEndpoints.Length == 0 ? true : (preferredAvailableReadEndpoints[0] != mostPreferredReadEndpoint); - - if (isPreferredListEmpty && endpointDiscoveryEnabled) - { - isMostPreferredLocationUnavailableForRead = preferredAvailableReadEndpoints[0] != accountLevelReadEndpoints[0]; - } - string mostPreferredWriteLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableWriteRegions[0] : this.preferredLocations.FirstOrDefault(location => databaseAccount.WritableRegions.Any(writeLocation => writeLocation.Name == location), ""); + if (isPreferredListEmpty && endpointDiscoveryEnabled) + { + isMostPreferredLocationUnavailableForRead = preferredAvailableReadEndpoints[0] != accountLevelReadEndpoints[0]; + } + + string mostPreferredWriteLocationName = (isPreferredListEmpty && endpointDiscoveryEnabled) ? preferredAvailableWriteRegions[0] : this.preferredLocations.FirstOrDefault(location => this.databaseAccount.WritableRegions.Any(writeLocation => writeLocation.Name == location), ""); Uri mostPreferredWriteEndpoint = LocationCacheTests.EndpointByLocation[mostPreferredWriteLocationName]; if (useMultipleWriteLocations) { isMostPreferredLocationUnavailableForWrite = preferredAvailableWriteEndpoints.Length == 0 ? true : (preferredAvailableWriteEndpoints[0] != mostPreferredWriteEndpoint); - } - - if (isPreferredListEmpty && endpointDiscoveryEnabled) - { - isMostPreferredLocationUnavailableForWrite = preferredAvailableWriteEndpoints[0] != accountLevelWriteEndpoints[0]; + } + + if (isPreferredListEmpty && endpointDiscoveryEnabled) + { + isMostPreferredLocationUnavailableForWrite = preferredAvailableWriteEndpoints[0] != accountLevelWriteEndpoints[0]; } } @@ -1324,11 +1329,11 @@ private void ValidateRequestEndpointResolution( firstAvailableReadEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; } - Uri firstWriteEnpoint = !endpointDiscoveryEnabled ? + Uri firstWriteEndpoint = !endpointDiscoveryEnabled ? LocationCacheTests.DefaultEndpoint : new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint); - Uri secondWriteEnpoint = !endpointDiscoveryEnabled ? + Uri secondWriteEndpoint = !endpointDiscoveryEnabled ? LocationCacheTests.DefaultEndpoint : new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); @@ -1340,10 +1345,11 @@ private void ValidateRequestEndpointResolution( Assert.AreEqual(firstAvailableWriteEndpoint, this.ResolveEndpointForWriteRequest(ResourceType.Document, false)); // Writes to other resource types should be directed to first/second write endpoint - Assert.AreEqual(firstWriteEnpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, false)); - Assert.AreEqual(secondWriteEnpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, true)); + Assert.AreEqual(firstWriteEndpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, false)); + Assert.AreEqual(secondWriteEndpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, true)); // Reads should be directed to available read endpoints regardless of resource type + //Console.WriteLine("Expected firstAvailableReadEndpoint : " + firstAvailableReadEndpoint + " Actual firstAvailableReadEndpoint : " + this.ResolveEndpointForReadRequest(true)); Assert.AreEqual(firstAvailableReadEndpoint, this.ResolveEndpointForReadRequest(true)); Assert.AreEqual(firstAvailableReadEndpoint, this.ResolveEndpointForReadRequest(false)); } From ab14fff78c8f88e47bd76bb1b6fc3e0550b82257 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 11 Sep 2024 19:37:14 -0400 Subject: [PATCH 05/16] Revert changes. --- Microsoft.Azure.Cosmos.sln | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index 03a17e2f39..bedaa04c34 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.10.35201.131 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29123.88 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject From 45b3f53060b8e663c2979a1a186131a848bfdde4 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 13 Sep 2024 14:49:44 -0400 Subject: [PATCH 06/16] Updated LocationCacheTests and CosmosAvailabilityStrategyTests. --- .../src/GatewayAccountReader.cs | 6 +- .../src/Routing/GlobalEndpointManager.cs | 63 +- .../src/Routing/LocationCache.cs | 1 - .../CosmosAvailabilityStrategyTests.cs | 176 +++-- .../GlobalEndpointManagerTest.cs | 30 +- .../LocationCacheTests.cs | 710 ++++++++++++------ 6 files changed, 667 insertions(+), 319 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index 1a2c3e14a4..20c795ea12 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos { using System; + using System.Collections.Generic; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -22,6 +23,7 @@ internal sealed class GatewayAccountReader private readonly CosmosHttpClient httpClient; private readonly Uri serviceEndpoint; private readonly CancellationToken cancellationToken; + private readonly ReaderWriterLockSlim readerWriterLock; // Backlog: Auth abstractions are spilling through. 4 arguments for this CTOR are result of it. public GatewayAccountReader(Uri serviceEndpoint, @@ -35,6 +37,7 @@ public GatewayAccountReader(Uri serviceEndpoint, this.cosmosAuthorization = cosmosAuthorization ?? throw new ArgumentNullException(nameof(AuthorizationTokenProvider)); this.connectionPolicy = connectionPolicy; this.cancellationToken = cancellationToken; + this.readerWriterLock = new ReaderWriterLockSlim(); } private async Task GetDatabaseAccountAsync(Uri serviceEndpoint) @@ -90,7 +93,8 @@ public async Task InitializeReaderAsync() locations: this.connectionPolicy.PreferredLocations, accountInitializationCustomEndpoints: this.connectionPolicy.AccountInitializationCustomEndpoints, getDatabaseAccountFn: this.GetDatabaseAccountAsync, - cancellationToken: this.cancellationToken); + cancellationToken: this.cancellationToken, + accountPropertiesReaderWriterLock: this.readerWriterLock); return databaseAccount; } diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 9cc27cb405..353fca8d7d 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -38,7 +38,7 @@ internal class GlobalEndpointManager : IGlobalEndpointManager private readonly int backgroundRefreshLocationTimeIntervalInMS = GlobalEndpointManager.DefaultBackgroundRefreshLocationTimeIntervalInMS; private readonly object backgroundAccountRefreshLock = new object(); private readonly object isAccountRefreshInProgressLock = new object(); - private readonly ReaderWriterLock locationCacheReadWriteLock = new ReaderWriterLock(); + private readonly ReaderWriterLockSlim locationCacheDatabaseAccountReadWriteLock = new ReaderWriterLockSlim(); private bool isAccountRefreshInProgress = false; private bool isBackgroundAccountRefreshActive = false; private DateTime LastBackgroundRefreshUtc = DateTime.MinValue; @@ -128,7 +128,8 @@ public static async Task GetDatabaseAccountFromAnyLocationsAs IList? locations, IList? accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + ReaderWriterLockSlim accountPropertiesReaderWriterLock) { using (GetAccountPropertiesHelper threadSafeGetAccountHelper = new GetAccountPropertiesHelper( defaultEndpoint, @@ -137,7 +138,7 @@ public static async Task GetDatabaseAccountFromAnyLocationsAs getDatabaseAccountFn, cancellationToken)) { - return await threadSafeGetAccountHelper.GetAccountPropertiesAsync(); + return await threadSafeGetAccountHelper.GetAccountPropertiesAsync(accountPropertiesReaderWriterLock); } } @@ -175,15 +176,15 @@ public GetAccountPropertiesHelper( .GetEnumerator(); } - public async Task GetAccountPropertiesAsync() + public async Task GetAccountPropertiesAsync(ReaderWriterLockSlim readerWriterLock) { // If there are no preferred regions or private endpoints, then just wait for the global endpoint results if (this.LimitToGlobalEndpointOnly) { - return await this.GetOnlyGlobalEndpointAsync(); + return await this.GetOnlyGlobalEndpointAsync(readerWriterLock); } - Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); + Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint, readerWriterLock); // Start a timer to start secondary requests in parallel. Task timerTask = Task.Delay(TimeSpan.FromSeconds(5)); @@ -203,8 +204,8 @@ public async Task GetAccountPropertiesAsync() HashSet tasksToWaitOn = new HashSet { globalEndpointTask, - this.TryGetAccountPropertiesFromAllLocationsAsync(), - this.TryGetAccountPropertiesFromAllLocationsAsync() + this.TryGetAccountPropertiesFromAllLocationsAsync(readerWriterLock), + this.TryGetAccountPropertiesFromAllLocationsAsync(readerWriterLock) }; while (tasksToWaitOn.Any()) @@ -236,14 +237,14 @@ public async Task GetAccountPropertiesAsync() throw new AggregateException(this.TransientExceptions); } - private async Task GetOnlyGlobalEndpointAsync() + private async Task GetOnlyGlobalEndpointAsync(ReaderWriterLockSlim readerWriterLock) { if (!this.LimitToGlobalEndpointOnly) { throw new ArgumentException("GetOnlyGlobalEndpointAsync should only be called if there are no other private endpoints or regions"); } - await this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); + await this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint, readerWriterLock); if (this.AccountProperties != null) { @@ -271,7 +272,7 @@ private async Task GetOnlyGlobalEndpointAsync() /// /// This is done in a thread safe way to allow multiple tasks to iterate over the list of service endpoints. /// - private async Task TryGetAccountPropertiesFromAllLocationsAsync() + private async Task TryGetAccountPropertiesFromAllLocationsAsync(ReaderWriterLockSlim readerWriterLock) { while (this.TryMoveNextServiceEndpointhreadSafe( out Uri? serviceEndpoint)) @@ -283,7 +284,8 @@ private async Task TryGetAccountPropertiesFromAllLocationsAsync() } await this.GetAndUpdateAccountPropertiesAsync( - endpoint: serviceEndpoint); + endpoint: serviceEndpoint, + readerWriterLock); } } @@ -317,7 +319,7 @@ private bool TryMoveNextServiceEndpointhreadSafe( } } - private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) + private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint, ReaderWriterLockSlim readerWriterLock) { try { @@ -335,8 +337,17 @@ private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) if (databaseAccount != null) { - this.AccountProperties = databaseAccount; - this.CancellationTokenSource.Cancel(); + readerWriterLock.EnterWriteLock(); + + try + { + this.AccountProperties = databaseAccount; + this.CancellationTokenSource.Cancel(); + } + finally + { + readerWriterLock.ExitWriteLock(); + } } } catch (Exception e) @@ -506,14 +517,14 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account return; } - this.locationCacheReadWriteLock.AcquireWriterLock(TimeSpan.FromMilliseconds(10)); + this.locationCacheDatabaseAccountReadWriteLock.EnterWriteLock(); try { this.locationCache.OnDatabaseAccountRead(databaseAccount); } finally { - this.locationCacheReadWriteLock.ReleaseWriterLock(); + this.locationCacheDatabaseAccountReadWriteLock.ExitWriteLock(); } if (this.isBackgroundAccountRefreshActive) @@ -653,7 +664,7 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh) AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true); - this.locationCacheReadWriteLock.AcquireWriterLock(TimeSpan.FromMilliseconds(10)); + this.locationCacheDatabaseAccountReadWriteLock.EnterWriteLock(); try { @@ -661,7 +672,7 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh) } finally { - this.locationCacheReadWriteLock.ReleaseWriterLock(); + this.locationCacheDatabaseAccountReadWriteLock.ExitWriteLock(); } } catch (Exception ex) @@ -686,10 +697,11 @@ internal async Task GetDatabaseAccountAsync(bool forceRefresh obsoleteValue: null, singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( this.defaultEndpoint, - this.connectionPolicy.PreferredLocations, + this.GetEffectivePreferredLocations(), this.connectionPolicy.AccountInitializationCustomEndpoints, this.GetDatabaseAccountAsync, - this.cancellationTokenSource.Token), + this.cancellationTokenSource.Token, + this.locationCacheDatabaseAccountReadWriteLock), cancellationToken: this.cancellationTokenSource.Token, forceRefresh: forceRefresh); #nullable enable @@ -713,20 +725,15 @@ private Collection GetEffectivePreferredLocations() return this.connectionPolicy.PreferredLocations; } - this.locationCacheReadWriteLock.AcquireReaderLock(TimeSpan.FromMilliseconds(10)); + this.locationCacheDatabaseAccountReadWriteLock.EnterReadLock(); try { - if (this.databaseAccountCache.Keys == null || this.databaseAccountCache.Keys.Count == 0) - { - return new Collection(); - } - return new Collection(this.locationCache.EffectivePreferredLocations); } finally { - this.locationCacheReadWriteLock.ReleaseReaderLock(); + this.locationCacheDatabaseAccountReadWriteLock.ExitReadLock(); } } } diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index 604de0b157..21477f93ca 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -10,7 +10,6 @@ namespace Microsoft.Azure.Cosmos.Routing using System.Collections.ObjectModel; using System.Linq; using Microsoft.Azure.Cosmos.Core.Trace; - using Microsoft.Azure.Cosmos.Linq; using Microsoft.Azure.Documents; /// diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 0bcf55481c..85eed019ff 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -158,8 +158,10 @@ public async Task TestInitAsync() } [TestMethod] + [DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")] + [DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")] [TestCategory("MultiRegion")] - public async Task AvailabilityStrategyNoTriggerTest() + public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmpty) { FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( id: "responseDely", @@ -197,7 +199,7 @@ public async Task AvailabilityStrategyNoTriggerTest() CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(300), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -232,8 +234,10 @@ public async Task AvailabilityStrategyNoTriggerTest() } [TestMethod] + [DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")] + [DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")] [TestCategory("MultiRegion")] - public async Task AvailabilityStrategyRequestOptionsTriggerTest() + public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferredLocationsEmpty) { FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( id: "responseDely", @@ -257,7 +261,7 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest() CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty? new List() : new List() { "Central US", "North Central US" }, Serializer = this.cosmosSystemTextJsonSerializer }; @@ -290,8 +294,10 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest() } [TestMethod] + [DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")] + [DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")] [TestCategory("MultiRegion")] - public async Task AvailabilityStrategyDisableOverideTest() + public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocationsEmpty) { FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( id: "responseDely", @@ -316,7 +322,7 @@ public async Task AvailabilityStrategyDisableOverideTest() CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -350,52 +356,97 @@ public async Task AvailabilityStrategyDisableOverideTest() [DataTestMethod] [TestCategory("MultiRegion")] - [DataRow("Read", "Read", "Gone", DisplayName = "Read | Gone")] - [DataRow("Read", "Read", "RetryWith", DisplayName = "Read | RetryWith")] - [DataRow("Read", "Read", "InternalServerError", DisplayName = "Read | InternalServerError")] - [DataRow("Read", "Read", "ReadSessionNotAvailable", DisplayName = "Read | ReadSessionNotAvailable")] - [DataRow("Read", "Read", "Timeout", DisplayName = "Read | Timeout")] - [DataRow("Read", "Read", "PartitionIsSplitting", DisplayName = "Read | PartitionIsSplitting")] - [DataRow("Read", "Read", "PartitionIsMigrating", DisplayName = "Read | PartitionIsMigrating")] - [DataRow("Read", "Read", "ServiceUnavailable", DisplayName = "Read | ServiceUnavailable")] - [DataRow("Read", "Read", "ResponseDelay", DisplayName = "Read | ResponseDelay")] - [DataRow("SinglePartitionQuery", "Query", "Gone", DisplayName = "SinglePartitionQuery | Gone")] - [DataRow("SinglePartitionQuery", "Query", "RetryWith", DisplayName = "SinglePartitionQuery | RetryWith")] - [DataRow("SinglePartitionQuery", "Query", "InternalServerError", DisplayName = "SinglePartitionQuery | InternalServerError")] - [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable")] - [DataRow("SinglePartitionQuery", "Query", "Timeout", DisplayName = "SinglePartitionQuery | Timeout")] - [DataRow("SinglePartitionQuery", "Query", "PartitionIsSplitting", DisplayName = "SinglePartitionQuery | PartitionIsSplitting")] - [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", DisplayName = "SinglePartitionQuery | PartitionIsMigrating")] - [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", DisplayName = "SinglePartitionQuery | ServiceUnavailable")] - [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", DisplayName = "SinglePartitionQuery | ResponseDelay")] - [DataRow("CrossPartitionQuery", "Query", "Gone", DisplayName = "CrossPartitionQuery | Gone")] - [DataRow("CrossPartitionQuery", "Query", "RetryWith", DisplayName = "CrossPartitionQuery | RetryWith")] - [DataRow("CrossPartitionQuery", "Query", "InternalServerError", DisplayName = "CrossPartitionQuery | InternalServerError")] - [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable")] - [DataRow("CrossPartitionQuery", "Query", "Timeout", DisplayName = "CrossPartitionQuery | Timeout")] - [DataRow("CrossPartitionQuery", "Query", "PartitionIsSplitting", DisplayName = "CrossPartitionQuery | PartitionIsSplitting")] - [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", DisplayName = "CrossPartitionQuery | PartitionIsMigrating")] - [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", DisplayName = "CrossPartitionQuery | ServiceUnavailable")] - [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", DisplayName = "CrossPartitionQuery | ResponseDelay")] - [DataRow("ReadMany", "ReadMany", "Gone", DisplayName = "ReadMany | Gone")] - [DataRow("ReadMany", "ReadMany", "RetryWith", DisplayName = "ReadMany | RetryWith")] - [DataRow("ReadMany", "ReadMany", "InternalServerError", DisplayName = "ReadMany | InternalServerError")] - [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", DisplayName = "ReadMany | ReadSessionNotAvailable")] - [DataRow("ReadMany", "ReadMany", "Timeout", DisplayName = "ReadMany | Timeout")] - [DataRow("ReadMany", "ReadMany", "PartitionIsSplitting", DisplayName = "ReadMany | PartitionIsSplitting")] - [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", DisplayName = "ReadMany | PartitionIsMigrating")] - [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", DisplayName = "ReadMany | ServiceUnavailable")] - [DataRow("ReadMany", "ReadMany", "ResponseDelay", DisplayName = "ReadMany | ResponseDelay")] - [DataRow("ChangeFeed", "ChangeFeed", "Gone", DisplayName = "ChangeFeed | Gone")] - [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", DisplayName = "ChangeFeed | RetryWith")] - [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", DisplayName = "ChangeFeed | InternalServerError")] - [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", DisplayName = "ChangeFeed | ReadSessionNotAvailable")] - [DataRow("ChangeFeed", "ChangeFeed", "Timeout", DisplayName = "ChangeFeed | Timeout")] - [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsSplitting", DisplayName = "ChangeFeed | PartitionIsSplitting")] - [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", DisplayName = "ChangeFeed | PartitionIsMigrating")] - [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", DisplayName = "ChangeFeed | ServiceUnavailable")] - [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", DisplayName = "ChangeFeed | ResponseDelay")] - public async Task AvailabilityStrategyAllFaultsTests(string operation, string conditonName, string resultName) + [DataRow("Read", "Read", "Gone", false, DisplayName = "Read | Gone | With Preferred Regions")] + [DataRow("Read", "Read", "RetryWith", false, DisplayName = "Read | RetryWith | With Preferred Regions")] + [DataRow("Read", "Read", "InternalServerError", false, DisplayName = "Read | InternalServerError | With Preferred Regions")] + [DataRow("Read", "Read", "ReadSessionNotAvailable", false, DisplayName = "Read | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("Read", "Read", "Timeout", false, DisplayName = "Read | Timeout | With Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsSplitting", false, DisplayName = "Read | PartitionIsSplitting | With Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsMigrating", false, DisplayName = "Read | PartitionIsMigrating | With Preferred Regions")] + [DataRow("Read", "Read", "ServiceUnavailable", false, DisplayName = "Read | ServiceUnavailable | With Preferred Regions")] + [DataRow("Read", "Read", "ResponseDelay", false, DisplayName = "Read | ResponseDelay | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Gone", false, DisplayName = "SinglePartitionQuery | Gone | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "RetryWith", false, DisplayName = "SinglePartitionQuery | RetryWith | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "InternalServerError", false, DisplayName = "SinglePartitionQuery | InternalServerError | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", false, DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Timeout", false, DisplayName = "SinglePartitionQuery | Timeout | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsSplitting", false, DisplayName = "SinglePartitionQuery | PartitionIsSplitting | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", false, DisplayName = "SinglePartitionQuery | PartitionIsMigrating | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", false, DisplayName = "SinglePartitionQuery | ServiceUnavailable | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", false, DisplayName = "SinglePartitionQuery | ResponseDelay | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Gone", false, DisplayName = "CrossPartitionQuery | Gone | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "RetryWith", false, DisplayName = "CrossPartitionQuery | RetryWith | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "InternalServerError", false, DisplayName = "CrossPartitionQuery | InternalServerError | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", false, DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Timeout", false, DisplayName = "CrossPartitionQuery | Timeout | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsSplitting", false, DisplayName = "CrossPartitionQuery | PartitionIsSplitting | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", false, DisplayName = "CrossPartitionQuery | PartitionIsMigrating | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", false, DisplayName = "CrossPartitionQuery | ServiceUnavailable | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", false, DisplayName = "CrossPartitionQuery | ResponseDelay | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Gone", false, DisplayName = "ReadMany | Gone | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "RetryWith", false, DisplayName = "ReadMany | RetryWith | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "InternalServerError", false, DisplayName = "ReadMany | InternalServerError | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", false, DisplayName = "ReadMany | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Timeout", false, DisplayName = "ReadMany | Timeout | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsSplitting", false, DisplayName = "ReadMany | PartitionIsSplitting | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", false, DisplayName = "ReadMany | PartitionIsMigrating | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", false, DisplayName = "ReadMany | ServiceUnavailable | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ResponseDelay", false, DisplayName = "ReadMany | ResponseDelay | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Gone", false, DisplayName = "ChangeFeed | Gone | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", false, DisplayName = "ChangeFeed | RetryWith | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", false, DisplayName = "ChangeFeed | InternalServerError | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", false, DisplayName = "ChangeFeed | ReadSessionNotAvailable | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Timeout", false, DisplayName = "ChangeFeed | Timeout | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsSplitting", false, DisplayName = "ChangeFeed | PartitionIsSplitting | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", false, DisplayName = "ChangeFeed | PartitionIsMigrating | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", false, DisplayName = "ChangeFeed | ServiceUnavailable | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", false, DisplayName = "ChangeFeed | ResponseDelay | With Preferred Regions")] + [DataRow("Read", "Read", "Gone", true, DisplayName = "Read | Gone | W/O Preferred Regions")] + [DataRow("Read", "Read", "RetryWith", true, DisplayName = "Read | RetryWith | W/O Preferred Regions")] + [DataRow("Read", "Read", "InternalServerError", true, DisplayName = "Read | InternalServerError | W/O Preferred Regions")] + [DataRow("Read", "Read", "ReadSessionNotAvailable", true, DisplayName = "Read | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("Read", "Read", "Timeout", true, DisplayName = "Read | Timeout | W/O Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsSplitting", true, DisplayName = "Read | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("Read", "Read", "PartitionIsMigrating", true, DisplayName = "Read | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("Read", "Read", "ServiceUnavailable", true, DisplayName = "Read | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("Read", "Read", "ResponseDelay", true, DisplayName = "Read | ResponseDelay | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Gone", true, DisplayName = "SinglePartitionQuery | Gone | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "RetryWith", true, DisplayName = "SinglePartitionQuery | RetryWith | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "InternalServerError", true, DisplayName = "SinglePartitionQuery | InternalServerError | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", true, DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "Timeout", true, DisplayName = "SinglePartitionQuery | Timeout | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsSplitting", true, DisplayName = "SinglePartitionQuery | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", true, DisplayName = "SinglePartitionQuery | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", true, DisplayName = "SinglePartitionQuery | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", true, DisplayName = "SinglePartitionQuery | ResponseDelay | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Gone", true, DisplayName = "CrossPartitionQuery | Gone | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "RetryWith", true, DisplayName = "CrossPartitionQuery | RetryWith | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "InternalServerError", true, DisplayName = "CrossPartitionQuery | InternalServerError | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", true, DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "Timeout", true, DisplayName = "CrossPartitionQuery | Timeout | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsSplitting", true, DisplayName = "CrossPartitionQuery | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", true, DisplayName = "CrossPartitionQuery | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", true, DisplayName = "CrossPartitionQuery | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", true, DisplayName = "CrossPartitionQuery | ResponseDelay | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Gone", true, DisplayName = "ReadMany | Gone | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "RetryWith", true, DisplayName = "ReadMany | RetryWith | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "InternalServerError", true, DisplayName = "ReadMany | InternalServerError | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", true, DisplayName = "ReadMany | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "Timeout", true, DisplayName = "ReadMany | Timeout | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsSplitting", true, DisplayName = "ReadMany | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", true, DisplayName = "ReadMany | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", true, DisplayName = "ReadMany | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ResponseDelay", true, DisplayName = "ReadMany | ResponseDelay | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Gone", true, DisplayName = "ChangeFeed | Gone | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", true, DisplayName = "ChangeFeed | RetryWith | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", true, DisplayName = "ChangeFeed | InternalServerError | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", true, DisplayName = "ChangeFeed | ReadSessionNotAvailable | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "Timeout", true, DisplayName = "ChangeFeed | Timeout | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsSplitting", true, DisplayName = "ChangeFeed | PartitionIsSplitting | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", true, DisplayName = "ChangeFeed | PartitionIsMigrating | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", true, DisplayName = "ChangeFeed | ServiceUnavailable | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", true, DisplayName = "ChangeFeed | ResponseDelay | W/O Preferred Regions")] + public async Task AvailabilityStrategyAllFaultsTests(string operation, string conditonName, string resultName, bool isPreferredLocationsEmpty) { FaultInjectionCondition conditon = this.conditions[conditonName]; IFaultInjectionResult result = this.results[resultName]; @@ -415,7 +466,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -562,12 +613,17 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co [DataTestMethod] [TestCategory("MultiRegion")] - [DataRow("Read", "Read", "ReadStep", DisplayName = "Read | ReadStep")] - [DataRow("SinglePartitionQuery", "Query", "QueryStep", DisplayName = "Query | SinglePartitionQueryStep")] - [DataRow("CrossPartitionQuery", "Query", "QueryStep", DisplayName = "Query | CrossPartitionQueryStep")] - [DataRow("ReadMany", "ReadMany", "ReadManyStep", DisplayName = "ReadMany | ReadManyStep")] - [DataRow("ChangeFeed", "ChangeFeed", "ChangeFeedStep", DisplayName = "ChangeFeed | ChangeFeedStep")] - public async Task AvailabilityStrategyStepTests(string operation, string conditonName1, string conditionName2) + [DataRow("Read", "Read", "ReadStep", false, DisplayName = "Read | ReadStep | With Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "QueryStep", false, DisplayName = "Query | SinglePartitionQueryStep | With Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "QueryStep", false, DisplayName = "Query | CrossPartitionQueryStep | With Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadManyStep", false, DisplayName = "ReadMany | ReadManyStep | With Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ChangeFeedStep", false, DisplayName = "ChangeFeed | ChangeFeedStep | With Preferred Regions")] + [DataRow("Read", "Read", "ReadStep", true, DisplayName = "Read | ReadStep | W/O Preferred Regions")] + [DataRow("SinglePartitionQuery", "Query", "QueryStep", true, DisplayName = "Query | SinglePartitionQueryStep | W/O Preferred Regions")] + [DataRow("CrossPartitionQuery", "Query", "QueryStep", true, DisplayName = "Query | CrossPartitionQueryStep | W/O Preferred Regions")] + [DataRow("ReadMany", "ReadMany", "ReadManyStep", true, DisplayName = "ReadMany | ReadManyStep | W/O Preferred Regions")] + [DataRow("ChangeFeed", "ChangeFeed", "ChangeFeedStep", true, DisplayName = "ChangeFeed | ChangeFeedStep | W/O Preferred Regions")] + public async Task AvailabilityStrategyStepTests(string operation, string conditonName1, string conditionName2, bool isPreferredRegionsEmpty) { FaultInjectionCondition conditon1 = this.conditions[conditonName1]; FaultInjectionCondition conditon2 = this.conditions[conditionName2]; @@ -596,7 +652,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = new List() { "Central US", "North Central US", "East US" }, + ApplicationPreferredRegions = isPreferredRegionsEmpty ? new List() : new List() { "Central US", "North Central US", "East US" }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index de89483a04..90c4c3ba8b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -120,7 +120,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( }, accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."), - cancellationTokenSource.Token); + cancellationTokenSource.Token, + new ReaderWriterLockSlim()); Assert.Fail("Previous call should have failed"); } @@ -159,7 +160,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( throw new Exception("This should never be hit since it should stop after the global endpoint hit the nonretriable exception"); }, - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.Fail("Should throw the UnauthorizedException"); } @@ -191,7 +193,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( throw new Microsoft.Azure.Documents.UnauthorizedException("Mock failed exception"); }, - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.Fail("Should throw the UnauthorizedException"); } @@ -222,7 +225,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( throw new Exception("This should never be hit since it should stop after the global endpoint hit the nonretriable exception"); }, - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.Fail("Should throw the ForbiddenException"); } @@ -251,7 +255,8 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( exceptions.Add(exception); throw exception; }, - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.Fail("Should throw the AggregateException"); } @@ -314,7 +319,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() }, accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.AreEqual(globalEndpointResult, databaseAccount); Assert.AreEqual(0, slowPrimaryRegionHelper.FailedEndpointCount); @@ -337,7 +343,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() }, accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); stopwatch.Stop(); Assert.AreEqual(globalEndpointResult, databaseAccount); @@ -361,7 +368,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() }, accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.AreEqual(globalEndpointResult, databaseAccount); Assert.AreEqual(3, slowPrimaryRegionHelper.FailedEndpointCount); @@ -383,7 +391,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() }, accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.AreEqual(globalEndpointResult, databaseAccount); Assert.AreEqual(0, slowPrimaryRegionHelper.FailedEndpointCount); @@ -409,7 +418,8 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() }, accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), - cancellationToken: default); + cancellationToken: default, + new ReaderWriterLockSlim()); Assert.AreEqual(globalEndpointResult, databaseAccount); Assert.AreEqual(5, slowPrimaryRegionHelper.FailedEndpointCount); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs index df3c92f7ed..fa536ecdc9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs @@ -27,6 +27,7 @@ namespace Microsoft.Azure.Cosmos.Client.Tests public sealed class LocationCacheTests { private static Uri DefaultEndpoint = new Uri("https://default.documents.azure.com"); + private static Uri DefaultRegionalEndpoint = new Uri("https://location1.documents.azure.com"); private static Uri Location1Endpoint = new Uri("https://location1.documents.azure.com"); private static Uri Location2Endpoint = new Uri("https://location2.documents.azure.com"); private static Uri Location3Endpoint = new Uri("https://location3.documents.azure.com"); @@ -48,23 +49,27 @@ public sealed class LocationCacheTests private Mock mockedClient; [TestMethod] + [DataRow(true, DisplayName = "Validate write endpoint order with preferred locations as non-empty and multi-write usage disabled.")] + [DataRow(false, DisplayName = "Validate write endpoint order with preferred locations as empty and multi-write usage disabled.")] [Owner("atulk")] - public void ValidateWriteEndpointOrderWithClientSideDisableMultipleWriteLocation() + public void ValidateWriteEndpointOrderWithClientSideDisableMultipleWriteLocation(bool isPreferredLocationListEmpty) { - using GlobalEndpointManager endpointManager = this.Initialize(false, true, false); + using GlobalEndpointManager endpointManager = this.Initialize(false, true, isPreferredLocationListEmpty); Assert.AreEqual(this.cache.WriteEndpoints[0], LocationCacheTests.Location1Endpoint); Assert.AreEqual(this.cache.WriteEndpoints[1], LocationCacheTests.Location2Endpoint); Assert.AreEqual(this.cache.WriteEndpoints[2], LocationCacheTests.Location3Endpoint); } [TestMethod] + [DataRow(true, DisplayName = "Validate get location with preferred locations as non-empty.")] + [DataRow(false, DisplayName = "Validate get location with preferred locations as empty.")] [Owner("atulk")] - public void ValidateGetLocation() + public void ValidateGetLocation(bool isPreferredLocationListEmpty) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: false, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: true); + isPreferredLocationsListEmpty: isPreferredLocationListEmpty); Assert.AreEqual(this.databaseAccount.WriteLocationsInternal.First().Name, this.cache.GetLocation(LocationCacheTests.DefaultEndpoint)); @@ -111,19 +116,19 @@ public void ValidateTryGetLocationForGatewayDiagnostics() [TestMethod] [Owner("atulk")] - public async Task ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryDisabled() + public async Task ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryDisabled() { - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, false, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, false, true); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, true, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(false, true, true); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, false, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, false, true); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, true, false); - await this.ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(true, true, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, false, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, false, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, true, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(false, true, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, false, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, false, true); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, true, false); + await this.ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(true, true, true); } - private async Task ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest) + private async Task ValidateRetryOnSessionNotAvailableWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest) { const bool enableEndpointDiscovery = false; @@ -188,13 +193,13 @@ private ClientRetryPolicy CreateClientRetryPolicy( [TestMethod] [Owner("atulk")] - public async Task ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabled() + public async Task ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabled() { - await this.ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(true); - await this.ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(false); + await this.ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(true); + await this.ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(false); } - private async Task ValidateRetryOnSessionNotAvailabeWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(bool isPreferredLocationsListEmpty) + private async Task ValidateRetryOnSessionNotAvailableWithDisableMultipleWriteLocationsAndEndpointDiscoveryEnabledAsync(bool isPreferredLocationsListEmpty) { const bool useMultipleWriteLocations = false; bool enableEndpointDiscovery = true; @@ -259,22 +264,23 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] + [DataRow(false, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry w/o preferredLocations")] + [DataRow(true, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry with preferredLocations")] [Owner("atulk")] - public async Task ValidateRetryOnReadSessionNotAvailabeWithEnableMultipleWriteLocationsAndEndpointDiscoveryEnabled() + public async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAndEndpointDiscoveryEnabled(bool isPreferredLocationsEmpty) { - await this.ValidateRetryOnReadSessionNotAvailabeWithEnableMultipleWriteLocationsAsync(); - await this.ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync(); + await this.ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty); + await this.ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty); } - private async Task ValidateRetryOnReadSessionNotAvailabeWithEnableMultipleWriteLocationsAsync() + private async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty) { const bool useMultipleWriteLocations = true; bool enableEndpointDiscovery = true; - ReadOnlyCollection preferredList = new List() { - "location2", - "location1" - }.AsReadOnly(); + ReadOnlyCollection preferredList = isPreferredLocationsEmpty + ? new List().AsReadOnly() + : new List() { "location2", "location1" }.AsReadOnly(); using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, @@ -285,72 +291,154 @@ private async Task ValidateRetryOnReadSessionNotAvailabeWithEnableMultipleWriteL endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: false, endpointManager); - using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) + if (!isPreferredLocationsEmpty) { - int retryCount = 0; - - try + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) { - await BackoffRetryUtility.ExecuteAsync( - () => - { - retryPolicy.OnBeforeSendRequest(request); + int retryCount = 0; - if (retryCount == 0) + try + { + await BackoffRetryUtility.ExecuteAsync( + () => { - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + retryPolicy.OnBeforeSendRequest(request); - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 1) - { - // Second request must go to the next preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + if (retryCount == 0) + { + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 2) - { - // Third request must go to first preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else - { - Assert.Fail(); - } + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the next preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; - retryCount++; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to first preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } - StoreResponseNameValueCollection headers = new(); - headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); - DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); - throw notFoundException; - }, - retryPolicy); + throw notFoundException; + }, + retryPolicy); - Assert.Fail(); + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(3, retryCount); + } } - catch (NotFoundException) + } + else + { + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations are the account-level read locations + Assert.AreEqual(4, effectivePreferredLocations.Count); + + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) { - DefaultTrace.TraceInformation("Received expected notFoundException"); - Assert.AreEqual(3, retryCount); + int retryCount = 0; + + try + { + await BackoffRetryUtility.ExecuteAsync( + () => + { + retryPolicy.OnBeforeSendRequest(request); + + if (retryCount == 0) + { + // First request must go to the first effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the second effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; + + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to third effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 3) + { + // Third request must go to fourth effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[3]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 4) + { + // Fourth request must go to first effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); + + throw notFoundException; + }, + retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(5, retryCount); + } } } + } - private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync() + private async Task ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty) { const bool useMultipleWriteLocations = true; bool enableEndpointDiscovery = true; - ReadOnlyCollection preferredList = new List() { - "location3", - "location2", - "location1" - }.AsReadOnly(); + ReadOnlyCollection preferredList = isPreferredLocationsEmpty + ? new List().AsReadOnly() + : new List() { "location3", "location2", "location1" }.AsReadOnly(); using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, @@ -361,78 +449,157 @@ private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWrite endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: false, endpointManager); - using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) + if (!isPreferredLocationsEmpty) { - int retryCount = 0; - - try + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { - await BackoffRetryUtility.ExecuteAsync( - () => - { - retryPolicy.OnBeforeSendRequest(request); + int retryCount = 0; - if (retryCount == 0) - { - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 1) - { - // Second request must go to the next preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 2) - { - // Third request must go to the next preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[2]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else if (retryCount == 3) - { - // Fourth request must go to first preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; - Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); - } - else + try + { + await BackoffRetryUtility.ExecuteAsync( + () => { - Assert.Fail(); - } + retryPolicy.OnBeforeSendRequest(request); - retryCount++; + if (retryCount == 0) + { + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the next preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to the next preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[2]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 3) + { + // Fourth request must go to first preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } - StoreResponseNameValueCollection headers = new(); - headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); - DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); + retryCount++; + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); - throw notFoundException; - }, - retryPolicy); - Assert.Fail(); + throw notFoundException; + }, + retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(4, retryCount); + } } - catch (NotFoundException) + } + else + { + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { - DefaultTrace.TraceInformation("Received expected notFoundException"); - Assert.AreEqual(4, retryCount); + int retryCount = 0; + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations are the account-level read locations + Assert.AreEqual(4, effectivePreferredLocations.Count); + + // for regions touched for writes - it will be the first 3 effectivePreferredLocations (location1, location2, location3) + // which are the write regions for the account + try + { + await BackoffRetryUtility.ExecuteAsync( + () => + { + retryPolicy.OnBeforeSendRequest(request); + + if (retryCount == 0) + { + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the next effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 2) + { + // Third request must go to the next effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 3) + { + // Fourth request must go to first effective preferred location + Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); + + + throw notFoundException; + }, + retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(4, retryCount); + } } } + } [TestMethod] + [DataRow(false, DisplayName = "Validate WriteForbidden retries with preferredLocations.")] + [DataRow(true, DisplayName = "Validate WriteForbidden retries w/o preferredLocations.")] [Owner("atulk")] - public async Task ValidateRetryOnWriteForbiddenExceptionAsync() + public async Task ValidateRetryOnWriteForbiddenExceptionAsync(bool isPreferredLocationsEmpty) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: false, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: false); + isPreferredLocationsListEmpty: isPreferredLocationsEmpty); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); + if (isPreferredLocationsEmpty) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { request.RequestContext.ResolvedPartitionKeyRange = new PartitionKeyRange() @@ -454,7 +621,9 @@ await BackoffRetryUtility.ExecuteAsync( { this.mockedClient.ResetCalls(); - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; + Uri expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]] : + LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); @@ -469,7 +638,10 @@ await BackoffRetryUtility.ExecuteAsync( this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny(), It.IsAny()), Times.Once); // Next request must go to next preferred endpoint - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + Uri expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]] : + LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); return Task.FromResult(true); @@ -486,21 +658,23 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] + [DataRow(false, DisplayName = "Validate DatabaseAccountNotFound retries with preferredLocations.")] + [DataRow(true, DisplayName = "Validate DatabaseAccountNotFound retries w/o preferredLocations.")] [Owner("atulk")] - public async Task ValidateRetryOnDatabaseAccountNotFoundAsync() + public async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool isPreferredLocationsEmpty) { - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: false); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: true); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: false); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: true); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty); } - private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest) + private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: enableMultipleWriteLocations, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: false); + isPreferredLocationsListEmpty: isPreferredLocationsEmpty); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); @@ -519,10 +693,14 @@ await BackoffRetryUtility.ExecuteAsync( retryCount++; retryPolicy.OnBeforeSendRequest(request); + // both retries check for flip-flop behavior b/w first two available write regions + // in case of multi-write enabled end to end (client + account) if (retryCount == 1) { - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; - + Uri expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]] : + LocationCacheTests.EndpointByLocation[this.preferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); StoreResponseNameValueCollection headers = new(); @@ -534,7 +712,10 @@ await BackoffRetryUtility.ExecuteAsync( else if (retryCount == 2) { // Next request must go to next preferred endpoint - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + Uri expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]] : + LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); return Task.FromResult(true); @@ -585,15 +766,17 @@ await this.ValidateLocationCacheAsync( } [TestMethod] - public async Task ValidateRetryOnHttpExceptionAsync() + [DataRow(false, DisplayName = "Validate retry on HTTP exception retries with preferredLocations.")] + [DataRow(true, DisplayName = "Validate retry on HTTP exception retries w/o preferredLocations.")] + public async Task ValidateRetryOnHttpExceptionAsync(bool isPreferredLocationsEmpty) { - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: false); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: true); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: false); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: true); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty); } - private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLocations, bool isReadRequest) + private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty) { ReadOnlyCollection preferredList = new List() { "location2", @@ -603,13 +786,19 @@ private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLoc using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: enableMultipleWriteLocations, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: false, + isPreferredLocationsListEmpty: isPreferredLocationsEmpty, preferedRegionListOverride: preferredList, enforceSingleMasterSingleWriteLocation: true); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); + if (isPreferredLocationsEmpty) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; @@ -629,7 +818,9 @@ await BackoffRetryUtility.ExecuteAsync( || isReadRequest) { // MultiMaster or Single Master Read can use preferred locations for first request - expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[0]]; + expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]] + : LocationCacheTests.EndpointByLocation[preferredList[0]]; } else { @@ -649,7 +840,9 @@ await BackoffRetryUtility.ExecuteAsync( || isReadRequest) { // Next request must go to next preferred endpoint - expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + expectedEndpoint = isPreferredLocationsEmpty ? + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]] + : LocationCacheTests.EndpointByLocation[preferredList[1]]; } else { @@ -678,18 +871,18 @@ await BackoffRetryUtility.ExecuteAsync( } [DataTestMethod] - [DataRow(true, false, false, false, false, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should NOT retry")] + [DataRow(true, false, false, true, false, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should retry")] [DataRow(false, false, false, false, false, DisplayName = "Write request - Single master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(true, true, false, false, false, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(false, true, false, false, false, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry")] + [DataRow(true, true, false, true, false, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should retry")] + [DataRow(false, true, false, true, false, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry")] [DataRow(true, false, true, true, false, DisplayName = "Read request - Single master - with preferred locations - without partition level failover - should retry")] [DataRow(false, false, true, false, false, DisplayName = "Write request - Single master - with preferred locations - without partition level failover - should NOT retry")] [DataRow(true, true, true, true, false, DisplayName = "Read request - Multi master - with preferred locations - without partition level failover - should retry")] [DataRow(false, true, true, true, false, DisplayName = "Write request - Multi master - with preferred locations - without partition level failover - should retry")] - [DataRow(true, false, false, false, true, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should NOT retry")] - [DataRow(false, false, false, false, true, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should NOT retry")] - [DataRow(true, true, false, false, true, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should NOT retry")] - [DataRow(false, true, false, false, true, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should NOT retry")] + [DataRow(true, false, false, true, true, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should retry")] + [DataRow(false, false, false, true, true, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should retry")] + [DataRow(true, true, false, true, true, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should retry")] + [DataRow(false, true, false, true, true, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should retry")] [DataRow(true, false, true, true, true, DisplayName = "Read request - Single master - with preferred locations - with partition level failover - should NOT retry")] [DataRow(false, false, true, true, true, DisplayName = "Write request - Single master - with preferred locations - with partition level failover - should retry")] [DataRow(true, true, true, true, true, DisplayName = "Read request - Multi master - with preferred locations - with partition level failover - should retry")] @@ -720,6 +913,13 @@ public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: enablePartitionLevelFailover, endpointManager); + if (!usesPreferredLocations) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; @@ -733,13 +933,67 @@ await BackoffRetryUtility.ExecuteAsync( if (retryCount == 1) { - if (!usesPreferredLocations) + Uri expectedEndpoint; + + if (usesPreferredLocations) { - Assert.Fail("Should not be retrying if preferredlocations is not being used"); + if (useMultipleWriteLocations) + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + } + else + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[preferredList[1]]; + } + } } - - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; - + else + { + if (useMultipleWriteLocations) + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + if (isReadRequest) + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + else + { + expectedEndpoint = + LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]]; + } + } + } + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount > 1) @@ -771,18 +1025,27 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] - [DataRow(true, true, true, DisplayName = "Read request - Multi master - with preferred locations")] - [DataRow(true, true, false, DisplayName = "Read request - Multi master - no preferred locations")] - [DataRow(true, false, true, DisplayName = "Read request - Single master - with preferred locations")] - [DataRow(true, false, false, DisplayName = "Read request - Single master - no preferred locations")] - [DataRow(false, true, true, DisplayName = "Write request - Multi master - with preferred locations")] - [DataRow(false, true, false, DisplayName = "Write request - Multi master - no preferred locations")] - [DataRow(false, false, true, DisplayName = "Write request - Single master - with preferred locations")] - [DataRow(false, false, false, DisplayName = "Write request - Single master - no preferred locations")] + [DataRow(true, true, true, false, DisplayName = "Read request - Multi master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, true, false, false, DisplayName = "Read request - Multi master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, false, true, false, DisplayName = "Read request - Single master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, false, false, false, DisplayName = "Read request - Single master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, true, true, false, DisplayName = "Write request - Multi master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, true, false, false, DisplayName = "Write request - Multi master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, false, true, false, DisplayName = "Write request - Single master - with preferred locations - default endpoint is not regional endpoint")] + [DataRow(false, false, false, false, DisplayName = "Write request - Single master - no preferred locations - default endpoint is not regional endpoint")] + [DataRow(true, true, true, true, DisplayName = "Read request - Multi master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(true, true, false, true, DisplayName = "Read request - Multi master - no preferred locations - default endpoint is regional endpoint")] + [DataRow(true, false, true, true, DisplayName = "Read request - Single master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(true, false, false, true, DisplayName = "Read request - Single master - no preferred locations - default endpoint is regional endpoint")] + [DataRow(false, true, true, true, DisplayName = "Write request - Multi master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(false, true, false, true, DisplayName = "Write request - Multi master - no preferred locations - default endpoint is regional endpoint")] + [DataRow(false, false, true, true, DisplayName = "Write request - Single master - with preferred locations - default endpoint is regional endpoint")] + [DataRow(false, false, false, true, DisplayName = "Write request - Single master - no preferred locations - default endpoint is regional endpoint")] public void VerifyRegionExcludedTest( bool isReadRequest, bool useMultipleWriteLocations, - bool usesPreferredLocations) + bool usesPreferredLocations, + bool isDefaultEndpointAlsoRegionEndpoint) { bool enableEndpointDiscovery = true; @@ -815,19 +1078,23 @@ public void VerifyRegionExcludedTest( List> excludeRegionCases = isReadRequest ? new List>() { - new List {"default"}, new List {"default", "location1"}, new List {"default", "location2"}, new List {"default", "location4"}, - new List {"default", "location1", "location2"}, new List {"default", "location1", "location4"}, new List {"default", "location2", "location4"}, - new List {"default", "location1", "location2", "location4"}, new List { "location1" }, new List {"location1", "location2"}, - new List {"location1", "location4"}, new List {"location1", "location2", "location4"},new List { "location2" }, - new List {"location2", "location4"},new List { "location4" }, - } : - new List>() + new List { "location1" }, + new List { "location2" }, + new List { "location4" }, + new List { "location1", "location2" }, + new List { "location1", "location4" }, + new List { "location2", "location4" }, + new List { "location1", "location2", "location4" }, + new List { "location1", "location2", "location3", "location4" }, + } : new List>() { - new List {"default" }, new List {"default", "location1"}, new List {"default", "location2"}, new List {"default", "location3"}, - new List {"default", "location1", "location2"}, new List {"default", "location1", "location3"}, new List {"default", "location2", "location3"}, - new List {"default", "location1", "location2", "location3"}, new List { "location1" }, new List {"location1", "location2"}, - new List {"location1", "location3"}, new List {"location1", "location2", "location3"},new List { "location2" }, - new List {"location2", "location3"},new List { "location3" }, + new List { "location1" }, + new List { "location2" }, + new List { "location3" }, + new List { "location1", "location2" }, + new List { "location1", "location3" }, + new List { "location2", "location3" }, + new List { "location1", "location2", "location3" } }; foreach (List excludeRegions in excludeRegionCases) @@ -835,10 +1102,11 @@ public void VerifyRegionExcludedTest( using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, - isPreferredLocationsListEmpty: false, + isPreferredLocationsListEmpty: !usesPreferredLocations, preferedRegionListOverride: preferredList, enforceSingleMasterSingleWriteLocation: true, - isExcludeRegionsTest: true); + isExcludeRegionsTest: true, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointAlsoRegionEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); @@ -846,11 +1114,25 @@ public void VerifyRegionExcludedTest( using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { request.RequestContext.ExcludeRegions = excludeRegions; + ReadOnlyCollection applicableEndpoints; - ReadOnlyCollection applicableEndpoints = this.cache.GetApplicableEndpoints(request, isReadRequest); + if (!isReadRequest && !useMultipleWriteLocations) + { + List applicableEndpointsInner = new List(1); + + Assert.IsNotNull(this.cache.WriteEndpoints); + Assert.IsTrue(this.cache.WriteEndpoints.Count > 0); + + applicableEndpointsInner.Add(this.cache.WriteEndpoints[0]); + applicableEndpoints = applicableEndpointsInner.AsReadOnly(); + } + else + { + applicableEndpoints = this.cache.GetApplicableEndpoints(request, isReadRequest); + } Uri endpoint = endpointManager.ResolveServiceEndpoint(request); - ReadOnlyCollection applicableRegions = this.GetApplicableRegions(isReadRequest, useMultipleWriteLocations, usesPreferredLocations, excludeRegions); + ReadOnlyCollection applicableRegions = this.GetApplicableRegions(isReadRequest, useMultipleWriteLocations, usesPreferredLocations, excludeRegions, isDefaultEndpointAlsoRegionEndpoint); Assert.AreEqual(applicableRegions.Count, applicableEndpoints.Count); for(int i = 0; i < applicableRegions.Count; i++) @@ -864,11 +1146,12 @@ public void VerifyRegionExcludedTest( } - private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, List excludeRegions) + private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, List excludeRegions, bool isDefaultEndpointARegionalEndpoint) { - if(!isReadRequest && !useMultipleWriteLocations) + // exclusion of write region for single-write maps to first available write region + if (!isReadRequest && !useMultipleWriteLocations) { - return new List() {LocationCacheTests.DefaultEndpoint }.AsReadOnly(); + return new List() { LocationCacheTests.Location1Endpoint }.AsReadOnly(); } Dictionary readWriteLocations = usesPreferredLocations ? @@ -888,42 +1171,52 @@ private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool us } : new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint }, } : isReadRequest ? new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint }, {"location1", LocationCacheTests.Location1Endpoint }, {"location2", LocationCacheTests.Location2Endpoint }, + {"location3", LocationCacheTests.Location3Endpoint }, {"location4", LocationCacheTests.Location4Endpoint }, } : useMultipleWriteLocations ? new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint }, {"location1", LocationCacheTests.Location1Endpoint }, {"location2", LocationCacheTests.Location2Endpoint }, {"location3", LocationCacheTests.Location3Endpoint }, } : new Dictionary() { - {"default", LocationCacheTests.DefaultEndpoint} }; List applicableRegions = new List(); - foreach (string region in readWriteLocations.Keys) + // exclude regions applies when + // 1. preferred regions are set + // 2. preferred regions aren't set and default endpoint isn't a regional endpoint + if (usesPreferredLocations || (!usesPreferredLocations && !isDefaultEndpointARegionalEndpoint)) { - if(!excludeRegions.Contains(region)) + foreach (string region in readWriteLocations.Keys) { - applicableRegions.Add(readWriteLocations[region]); + if (!excludeRegions.Contains(region)) + { + applicableRegions.Add(readWriteLocations[region]); + } } - } - + } + if (applicableRegions.Count == 0) { - applicableRegions.Add(LocationCacheTests.DefaultEndpoint); + if (isDefaultEndpointARegionalEndpoint) + { + applicableRegions.Add(LocationCacheTests.DefaultRegionalEndpoint); + } + else + { + applicableRegions.Add(LocationCacheTests.DefaultEndpoint); + } } return applicableRegions.AsReadOnly(); @@ -934,16 +1227,7 @@ private static AccountProperties CreateDatabaseAccount( bool enforceSingleMasterSingleWriteLocation, bool isExcludeRegionsTest = false) { - Collection writeLocations = isExcludeRegionsTest ? - - new Collection() - { - { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, - { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, - { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, - { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, - } : - new Collection() + Collection writeLocations = new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, @@ -955,11 +1239,7 @@ private static AccountProperties CreateDatabaseAccount( { // Some pre-existing tests depend on the account having multiple write locations even on single master setup // Newer tests can correctly define a single master account (single write region) without breaking existing tests - writeLocations = isExcludeRegionsTest ? - new Collection() - { - { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } } - } : + writeLocations = new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } } @@ -969,16 +1249,8 @@ private static AccountProperties CreateDatabaseAccount( AccountProperties databaseAccount = new AccountProperties() { EnableMultipleWriteLocations = useMultipleWriteLocations, - ReadLocationsInternal = isExcludeRegionsTest ? - new Collection() - { - { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, - { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, - { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, - { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, - { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, - } : - new Collection() + // ReadLocations should be a superset of WriteLocations + ReadLocationsInternal = new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, @@ -998,7 +1270,8 @@ private GlobalEndpointManager Initialize( bool enforceSingleMasterSingleWriteLocation = false, // Some tests depend on the Initialize to create an account with multiple write locations, even when not multi master ReadOnlyCollection preferedRegionListOverride = null, bool enablePartitionLevelFailover = false, - bool isExcludeRegionsTest = false) + bool isExcludeRegionsTest = false, + bool isDefaultEndpointARegionalEndpoint = false) { this.databaseAccount = LocationCacheTests.CreateDatabaseAccount( useMultipleWriteLocations, @@ -1022,7 +1295,7 @@ private GlobalEndpointManager Initialize( this.cache = new LocationCache( this.preferredLocations, - LocationCacheTests.DefaultEndpoint, + isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint, enableEndpointDiscovery, 10, useMultipleWriteLocations); @@ -1030,7 +1303,7 @@ private GlobalEndpointManager Initialize( this.cache.OnDatabaseAccountRead(this.databaseAccount); this.mockedClient = new Mock(); - this.mockedClient.Setup(owner => owner.ServiceEndpoint).Returns(LocationCacheTests.DefaultEndpoint); + this.mockedClient.Setup(owner => owner.ServiceEndpoint).Returns(isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint); this.mockedClient.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny(), It.IsAny())).ReturnsAsync(this.databaseAccount); ConnectionPolicy connectionPolicy = new ConnectionPolicy() @@ -1045,7 +1318,7 @@ private GlobalEndpointManager Initialize( } GlobalEndpointManager endpointManager = new GlobalEndpointManager(this.mockedClient.Object, connectionPolicy); - + this.partitionKeyRangeLocationCache = enablePartitionLevelFailover ? new GlobalPartitionEndpointManagerCore(endpointManager) : GlobalPartitionEndpointManagerNoOp.Instance; @@ -1349,7 +1622,6 @@ private void ValidateRequestEndpointResolution( Assert.AreEqual(secondWriteEndpoint, this.ResolveEndpointForWriteRequest(ResourceType.Database, true)); // Reads should be directed to available read endpoints regardless of resource type - //Console.WriteLine("Expected firstAvailableReadEndpoint : " + firstAvailableReadEndpoint + " Actual firstAvailableReadEndpoint : " + this.ResolveEndpointForReadRequest(true)); Assert.AreEqual(firstAvailableReadEndpoint, this.ResolveEndpointForReadRequest(true)); Assert.AreEqual(firstAvailableReadEndpoint, this.ResolveEndpointForReadRequest(false)); } From c6758d8f0b3338de7dcde44aea2ad8d8bc35337f Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 13 Sep 2024 14:54:37 -0400 Subject: [PATCH 07/16] Revert changes. --- Microsoft.Azure.Cosmos.sln | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index bedaa04c34..d412905195 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -164,18 +164,6 @@ Global {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|Any CPU.Build.0 = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.ActiveCfg = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.Build.0 = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 8c14ad40dc92beaa76e3d4cbcfa57b01fee42af7 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 13 Sep 2024 15:45:17 -0400 Subject: [PATCH 08/16] Fixing tests. --- .../CosmosAvailabilityStrategyTests.cs | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 85eed019ff..32173e79bd 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -227,9 +227,20 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp Assert.IsNotNull(hedgeContext); IReadOnlyCollection hedgeContextList; hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.AreEqual(2, hedgeContextList.Count); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); + + if (isPreferredLocationsEmpty) + { + Assert.AreEqual(3, hedgeContextList.Count); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.eastUs)); + } + else + { + Assert.AreEqual(2, hedgeContextList.Count); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); + Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); + } }; } @@ -401,7 +412,7 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", false, DisplayName = "ChangeFeed | PartitionIsMigrating | With Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", false, DisplayName = "ChangeFeed | ServiceUnavailable | With Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", false, DisplayName = "ChangeFeed | ResponseDelay | With Preferred Regions")] - [DataRow("Read", "Read", "Gone", true, DisplayName = "Read | Gone | W/O Preferred Regions")] + [DataRow("Read", "Read", "Gone", true, DisplayName = "Read | Gone | W/O Preferred Regions")] [DataRow("Read", "Read", "RetryWith", true, DisplayName = "Read | RetryWith | W/O Preferred Regions")] [DataRow("Read", "Read", "InternalServerError", true, DisplayName = "Read | InternalServerError | W/O Preferred Regions")] [DataRow("Read", "Read", "ReadSessionNotAvailable", true, DisplayName = "Read | ReadSessionNotAvailable | W/O Preferred Regions")] @@ -488,9 +499,17 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co case "Read": rule.Enable(); + ItemRequestOptions itemRequestOptions = new ItemRequestOptions(); + + if (isPreferredLocationsEmpty) + { + itemRequestOptions.ExcludeRegions = new List() { "East US" }; + } + ItemResponse ir = await container.ReadItemAsync( "testId", - new PartitionKey("pk")); + new PartitionKey("pk"), + itemRequestOptions); Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; @@ -509,6 +528,11 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co PartitionKey = new PartitionKey("pk"), }; + if (isPreferredLocationsEmpty) + { + requestOptions.ExcludeRegions = new List() { "East US" }; + } + FeedIterator queryIterator = container.GetItemQueryIterator( new QueryDefinition(queryString), requestOptions: requestOptions); @@ -531,8 +555,18 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co case "CrossPartitionQuery": string crossPartitionQueryString = "SELECT * FROM c"; + + QueryRequestOptions queryRequestOptions = new QueryRequestOptions(); + + if (isPreferredLocationsEmpty) + { + queryRequestOptions.ExcludeRegions = new List() { "East US" }; + } + FeedIterator crossPartitionQueryIterator = container.GetItemQueryIterator( - new QueryDefinition(crossPartitionQueryString)); + new QueryDefinition(crossPartitionQueryString), + null, + queryRequestOptions); rule.Enable(); @@ -553,6 +587,13 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co case "ReadMany": rule.Enable(); + ReadManyRequestOptions readManyRequestOptions = new ReadManyRequestOptions(); + + if (isPreferredLocationsEmpty) + { + readManyRequestOptions.ExcludeRegions = new List() { "East US" }; + } + FeedResponse readManyResponse = await container.ReadManyItemsAsync( new List<(string, PartitionKey)>() { @@ -560,7 +601,8 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co ("testId2", new PartitionKey("pk2")), ("testId3", new PartitionKey("pk3")), ("testId4", new PartitionKey("pk4")) - }); + }, + readManyRequestOptions); Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; From 98c0a67e88567003a263a9043644a642f08996ed Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 13 Sep 2024 17:28:47 -0400 Subject: [PATCH 09/16] Additional wiring of effective preferred regions. --- Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs | 2 +- .../src/Telemetry/TelemetryToServiceHelper.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 353fca8d7d..6cc829cd3d 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -718,7 +718,7 @@ private bool SkipRefresh(bool forceRefresh) && !forceRefresh; } - private Collection GetEffectivePreferredLocations() + public Collection GetEffectivePreferredLocations() { if (this.connectionPolicy.PreferredLocations != null && this.connectionPolicy.PreferredLocations.Count > 0) { diff --git a/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs b/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs index 4b94811b67..2f655c2032 100644 --- a/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs +++ b/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs @@ -206,7 +206,7 @@ private void InitializeClientTelemetry(AccountClientConfiguration clientConfig) connectionMode: this.connectionPolicy.ConnectionMode, authorizationTokenProvider: this.cosmosAuthorization, diagnosticsHelper: DiagnosticsHandlerHelper.GetInstance(), - preferredRegions: this.connectionPolicy.PreferredLocations, + preferredRegions: this.globalEndpointManager.GetEffectivePreferredLocations(), globalEndpointManager: this.globalEndpointManager, endpointUrl: clientConfig.ClientTelemetryConfiguration.Endpoint); From 31744c270099e72636e515a5c6abf32f7dbaddce Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 13 Sep 2024 18:23:34 -0400 Subject: [PATCH 10/16] Modified GlobalEndpointManagerTest.cs --- .../GlobalEndpointManagerTest.cs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index 90c4c3ba8b..a3ae4b8d25 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -427,6 +427,67 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() } } + /// + /// Tests for + /// + [TestMethod] + public async Task GetDatabaseAccountFromEffectiveRegionalEndpointTestAsync() + { + AccountProperties databaseAccount = new AccountProperties + { + ReadLocationsInternal = new Collection() + { + new AccountRegion + { + Name = "Location1", + Endpoint = "https://testfailover-location1.documents-test.windows-int.net/" + }, + new AccountRegion + { + Name = "Location2", + Endpoint = "https://testfailover-location2.documents-test.windows-int.net/" + }, + new AccountRegion + { + Name = "Location3", + Endpoint = "https://testfailover-location3.documents-test.windows-int.net/" + }, + } + }; + + Uri defaultEndpoint = new Uri("https://testfailover.documents-test.windows-int.net/"); + Uri effectivePreferredRegion1SuffixedUri = new Uri("https://testfailover-location1.documents-test.windows-int.net/"); + + //Setup mock owner "document client" + Mock mockOwner = new Mock(); + + mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(defaultEndpoint); + mockOwner.SetupSequence(owner => + owner.GetDatabaseAccountInternalAsync(defaultEndpoint, It.IsAny())) + .ReturnsAsync(databaseAccount) + .ThrowsAsync(new HttpRequestException()); + mockOwner.Setup(owner => + owner.GetDatabaseAccountInternalAsync(effectivePreferredRegion1SuffixedUri, It.IsAny())) + .ReturnsAsync(databaseAccount); + + // Create connection policy with no preferred locations + ConnectionPolicy connectionPolicy = new ConnectionPolicy(); + + using GlobalEndpointManager globalEndpointManager = + new GlobalEndpointManager(mockOwner.Object, connectionPolicy); + globalEndpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(databaseAccount); + + await Task.Delay(TimeSpan.FromSeconds(5)); + await globalEndpointManager.RefreshLocationAsync(forceRefresh: true); + + mockOwner.Verify( + owner => owner.GetDatabaseAccountInternalAsync(defaultEndpoint, It.IsAny()), + Times.Exactly(2)); + mockOwner.Verify( + owner => owner.GetDatabaseAccountInternalAsync(effectivePreferredRegion1SuffixedUri, It.IsAny()), + Times.Once); + } + /// /// Test to validate that when an exception is thrown during a RefreshLocationAsync call /// the exception should not be bubbled up and remain unobserved. The exception should be From 6ca25a713496fc40a6dc32b2ca48c1cffa73c1cd Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Sun, 15 Sep 2024 21:37:19 -0400 Subject: [PATCH 11/16] Modify LocationCacheTests.cs --- .../src/Routing/LocationCache.cs | 8 + .../LocationCacheTests.cs | 536 ++++++++++++++---- 2 files changed, 421 insertions(+), 123 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index 21477f93ca..7c6a2ae830 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -253,6 +253,14 @@ public ReadOnlyDictionary GetAvailableReadEndpointsByLocation() return this.locationInfo.AvailableReadEndpointByLocation; } + /// + /// Gets account-level write locations. + /// + public ReadOnlyCollection GetAvailableWriteLocations() + { + return this.locationInfo.AvailableWriteLocations; + } + public Uri GetHubUri() { DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs index fa536ecdc9..df9251fb76 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs @@ -49,12 +49,19 @@ public sealed class LocationCacheTests private Mock mockedClient; [TestMethod] - [DataRow(true, DisplayName = "Validate write endpoint order with preferred locations as non-empty and multi-write usage disabled.")] - [DataRow(false, DisplayName = "Validate write endpoint order with preferred locations as empty and multi-write usage disabled.")] + [DataRow(true, false, DisplayName = "Validate write endpoint order with preferred locations as empty and multi-write usage disabled and default endpoint is global endpoint.")] + [DataRow(false, false, DisplayName = "Validate write endpoint order with preferred locations as non-empty and multi-write usage disabled and default endpoint is global endpoint.")] + [DataRow(true, true, DisplayName = "Validate write endpoint order with preferred locations as empty and multi-write usage disabled and default endpoint is regional endpoint.")] + [DataRow(false, true, DisplayName = "Validate write endpoint order with preferred locations as non-empty and multi-write usage disabled and default endpoint is regional endpoint.")] [Owner("atulk")] - public void ValidateWriteEndpointOrderWithClientSideDisableMultipleWriteLocation(bool isPreferredLocationListEmpty) + public void ValidateWriteEndpointOrderWithClientSideDisableMultipleWriteLocation(bool isPreferredLocationListEmpty, bool isDefaultEndpointARegionalEndpoint) { - using GlobalEndpointManager endpointManager = this.Initialize(false, true, isPreferredLocationListEmpty); + using GlobalEndpointManager endpointManager = this.Initialize( + useMultipleWriteLocations: false, + enableEndpointDiscovery: true, + isPreferredLocationsListEmpty: isPreferredLocationListEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); + Assert.AreEqual(this.cache.WriteEndpoints[0], LocationCacheTests.Location1Endpoint); Assert.AreEqual(this.cache.WriteEndpoints[1], LocationCacheTests.Location2Endpoint); Assert.AreEqual(this.cache.WriteEndpoints[2], LocationCacheTests.Location3Endpoint); @@ -264,16 +271,18 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] - [DataRow(false, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry w/o preferredLocations")] - [DataRow(true, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry with preferredLocations")] + [DataRow(false, false, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry w/o preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry with preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry w/o preferredLocations with regional default endpoint.")] + [DataRow(true, true, DisplayName = "Validate (Read/Write)SessionNotAvailable cross-region retry with preferredLocations with regional default endpoint.")] [Owner("atulk")] - public async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAndEndpointDiscoveryEnabled(bool isPreferredLocationsEmpty) + public async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAndEndpointDiscoveryEnabled(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { - await this.ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty); - await this.ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty); + await this.ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); } - private async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty) + private async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { const bool useMultipleWriteLocations = true; bool enableEndpointDiscovery = true; @@ -286,7 +295,8 @@ private async Task ValidateRetryOnReadSessionNotAvailableWithEnableMultipleWrite useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, isPreferredLocationsListEmpty: false, - preferedRegionListOverride: preferredList); + preferedRegionListOverride: preferredList, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: false, endpointManager); @@ -353,53 +363,59 @@ await BackoffRetryUtility.ExecuteAsync( } else { - ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; - - // effective preferred locations are the account-level read locations - Assert.AreEqual(4, effectivePreferredLocations.Count); - - using (DocumentServiceRequest request = - this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) + if (!isDefaultEndpointARegionalEndpoint) { - int retryCount = 0; + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; - try + // effective preferred locations are the account-level read locations + Assert.AreEqual(4, effectivePreferredLocations.Count); + + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) { - await BackoffRetryUtility.ExecuteAsync( - () => + int retryCount = 0; + + try + { + await BackoffRetryUtility.ExecuteAsync(() => { retryPolicy.OnBeforeSendRequest(request); if (retryCount == 0) { // First request must go to the first effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 1) { // Second request must go to the second effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 2) { // Third request must go to third effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 3) { // Third request must go to fourth effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[3]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[3]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 4) { // Fourth request must go to first effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else @@ -414,24 +430,82 @@ await BackoffRetryUtility.ExecuteAsync( ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); - + throw notFoundException; - }, - retryPolicy); + }, retryPolicy); - Assert.Fail(); + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(5, retryCount); + } } - catch (NotFoundException) + } + else + { + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations is just the default regional endpoint + Assert.AreEqual(1, effectivePreferredLocations.Count); + + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: true, isMasterResourceType: false)) { - DefaultTrace.TraceInformation("Received expected notFoundException"); - Assert.AreEqual(5, retryCount); + int retryCount = 0; + + try + { + await BackoffRetryUtility.ExecuteAsync(() => + { + retryPolicy.OnBeforeSendRequest(request); + + if (retryCount == 0) + { + // First request must go to the first effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the second effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); + + throw notFoundException; + }, retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(2, retryCount); + } } } } - } - private async Task ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty) + private async Task ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWriteLocationsAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { const bool useMultipleWriteLocations = true; bool enableEndpointDiscovery = true; @@ -444,7 +518,8 @@ private async Task ValidateRetryOnWriteSessionNotAvailableWithEnableMultipleWrit useMultipleWriteLocations: useMultipleWriteLocations, enableEndpointDiscovery: enableEndpointDiscovery, isPreferredLocationsListEmpty: false, - preferedRegionListOverride: preferredList); + preferedRegionListOverride: preferredList, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery, partitionLevelFailoverEnabled: false, endpointManager); @@ -512,44 +587,50 @@ await BackoffRetryUtility.ExecuteAsync( } else { - using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) + if (!isDefaultEndpointARegionalEndpoint) { - int retryCount = 0; - ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; - - // effective preferred locations are the account-level read locations - Assert.AreEqual(4, effectivePreferredLocations.Count); - - // for regions touched for writes - it will be the first 3 effectivePreferredLocations (location1, location2, location3) - // which are the write regions for the account - try + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { - await BackoffRetryUtility.ExecuteAsync( - () => + int retryCount = 0; + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations are the account-level read locations + Assert.AreEqual(4, effectivePreferredLocations.Count); + + // for regions touched for writes - it will be the first 3 effectivePreferredLocations (location1, location2, location3) + // which are the write regions for the account + try + { + await BackoffRetryUtility.ExecuteAsync(() => { retryPolicy.OnBeforeSendRequest(request); if (retryCount == 0) { - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 1) { // Second request must go to the next effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[1]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 2) { // Third request must go to the next effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[2]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else if (retryCount == 3) { // Fourth request must go to first effective preferred location - Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); } else @@ -560,44 +641,113 @@ await BackoffRetryUtility.ExecuteAsync( retryCount++; StoreResponseNameValueCollection headers = new(); - headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); - DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers); - + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); throw notFoundException; - }, - retryPolicy); + }, retryPolicy); - Assert.Fail(); + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(4, retryCount); + } } - catch (NotFoundException) + } + else + { + using (DocumentServiceRequest request = + this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) { - DefaultTrace.TraceInformation("Received expected notFoundException"); - Assert.AreEqual(4, retryCount); + int retryCount = 0; + ReadOnlyCollection effectivePreferredLocations = this.cache.EffectivePreferredLocations; + + // effective preferred locations is just the default regional endpoint + Assert.AreEqual(1, effectivePreferredLocations.Count); + + // for regions touched for writes - it will be the first 3 effectivePreferredLocations (location1, location2, location3) + // which are the write regions for the account + try + { + await BackoffRetryUtility.ExecuteAsync(() => + { + retryPolicy.OnBeforeSendRequest(request); + + if (retryCount == 0) + { + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else if (retryCount == 1) + { + // Second request must go to the first effective preferred location + Uri expectedEndpoint = + LocationCacheTests.EndpointByLocation[effectivePreferredLocations[0]]; + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); + } + else + { + Assert.Fail(); + } + + retryCount++; + + StoreResponseNameValueCollection headers = new(); + headers[WFConstants.BackendHeaders.SubStatus] = + ((int)SubStatusCodes.ReadSessionNotAvailable).ToString(); + DocumentClientException notFoundException = + new NotFoundException(RMResources.NotFound, headers); + + throw notFoundException; + }, retryPolicy); + + Assert.Fail(); + } + catch (NotFoundException) + { + DefaultTrace.TraceInformation("Received expected notFoundException"); + Assert.AreEqual(2, retryCount); + } } } } - } [TestMethod] - [DataRow(false, DisplayName = "Validate WriteForbidden retries with preferredLocations.")] - [DataRow(true, DisplayName = "Validate WriteForbidden retries w/o preferredLocations.")] + [DataRow(false, false, DisplayName = "Validate WriteForbidden retries with preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate WriteForbidden retries w/o preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate WriteForbidden retries with preferredLocations with regional default endpoint.")] + [DataRow(true, true, DisplayName = "Validate WriteForbidden retries w/o preferredLocations with regional default endpoint.")] [Owner("atulk")] - public async Task ValidateRetryOnWriteForbiddenExceptionAsync(bool isPreferredLocationsEmpty) + public async Task ValidateRetryOnWriteForbiddenExceptionAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: false, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: isPreferredLocationsEmpty); + isPreferredLocationsListEmpty: isPreferredLocationsEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); if (isPreferredLocationsEmpty) { - Assert.IsNotNull(this.cache.EffectivePreferredLocations); - Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + if (!isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } } using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false)) @@ -637,10 +787,33 @@ await BackoffRetryUtility.ExecuteAsync( { this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny(), It.IsAny()), Times.Once); - // Next request must go to next preferred endpoint - Uri expectedEndpoint = isPreferredLocationsEmpty ? - LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]] : - LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + // Next request must go to next available write endpoint + Uri expectedEndpoint; + + if (isPreferredLocationsEmpty) + { + if (isDefaultEndpointARegionalEndpoint) + { + ReadOnlyCollection availableWriteLocations = + this.cache.GetAvailableWriteLocations(); + + Assert.IsNotNull(availableWriteLocations); + Assert.AreEqual(3, availableWriteLocations.Count); + + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(this.cache.EffectivePreferredLocations.Count, 1); + + expectedEndpoint = LocationCacheTests.EndpointByLocation[availableWriteLocations[1]]; + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + } Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); @@ -658,29 +831,54 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] - [DataRow(false, DisplayName = "Validate DatabaseAccountNotFound retries with preferredLocations.")] - [DataRow(true, DisplayName = "Validate DatabaseAccountNotFound retries w/o preferredLocations.")] + [DataRow(false, false, DisplayName = "Validate DatabaseAccountNotFound retries with preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate DatabaseAccountNotFound retries w/o preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate DatabaseAccountNotFound retries with preferredLocations with global default endpoint.")] + [DataRow(true, true, DisplayName = "Validate DatabaseAccountNotFound retries w/o preferredLocations with global default endpoint.")] [Owner("atulk")] - public async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool isPreferredLocationsEmpty) + public async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty); - await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnDatabaseAccountNotFoundAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); } - private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty) + private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { using GlobalEndpointManager endpointManager = this.Initialize( useMultipleWriteLocations: enableMultipleWriteLocations, enableEndpointDiscovery: true, - isPreferredLocationsListEmpty: isPreferredLocationsEmpty); + isPreferredLocationsListEmpty: isPreferredLocationsEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); + if (isPreferredLocationsEmpty) + { + if (enableMultipleWriteLocations) + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.IsTrue(this.cache.EffectivePreferredLocations.Count == 1); + Assert.IsTrue(this.cache.EffectivePreferredLocations[0] == "location1"); + } + } + else + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.IsTrue(this.cache.EffectivePreferredLocations.Count == 1); + Assert.IsTrue(this.cache.EffectivePreferredLocations[0] == "location1"); + } + } + } + endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1; - + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; @@ -711,11 +909,34 @@ await BackoffRetryUtility.ExecuteAsync( } else if (retryCount == 2) { - // Next request must go to next preferred endpoint - Uri expectedEndpoint = isPreferredLocationsEmpty ? - LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]] : - LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; - + // Next request must go to next available write endpoint + Uri expectedEndpoint; + + if (isPreferredLocationsEmpty) + { + if (isDefaultEndpointARegionalEndpoint) + { + ReadOnlyCollection availableWriteLocations = + this.cache.GetAvailableWriteLocations(); + + Assert.IsNotNull(availableWriteLocations); + Assert.AreEqual(3, availableWriteLocations.Count); + + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(this.cache.EffectivePreferredLocations.Count, 1); + + expectedEndpoint = LocationCacheTests.EndpointByLocation[availableWriteLocations[1]]; + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]]; + } + Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute); return Task.FromResult(true); @@ -766,17 +987,19 @@ await this.ValidateLocationCacheAsync( } [TestMethod] - [DataRow(false, DisplayName = "Validate retry on HTTP exception retries with preferredLocations.")] - [DataRow(true, DisplayName = "Validate retry on HTTP exception retries w/o preferredLocations.")] - public async Task ValidateRetryOnHttpExceptionAsync(bool isPreferredLocationsEmpty) + [DataRow(false, false, DisplayName = "Validate retry on HTTP exception retries with preferredLocations with global default endpoint.")] + [DataRow(true, false, DisplayName = "Validate retry on HTTP exception retries w/o preferredLocations with global default endpoint.")] + [DataRow(false, true, DisplayName = "Validate retry on HTTP exception retries with preferredLocations with regional default endpoint.")] + [DataRow(true, true, DisplayName = "Validate retry on HTTP exception retries w/o preferredLocations with regional default endpoint.")] + public async Task ValidateRetryOnHttpExceptionAsync(bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty); - await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: false, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: false, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); + await this.ValidateRetryOnHttpExceptionAsync(enableMultipleWriteLocations: true, isReadRequest: true, isPreferredLocationsEmpty, isDefaultEndpointARegionalEndpoint); } - private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty) + private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLocations, bool isReadRequest, bool isPreferredLocationsEmpty, bool isDefaultEndpointARegionalEndpoint) { ReadOnlyCollection preferredList = new List() { "location2", @@ -788,17 +1011,44 @@ private async Task ValidateRetryOnHttpExceptionAsync(bool enableMultipleWriteLoc enableEndpointDiscovery: true, isPreferredLocationsListEmpty: isPreferredLocationsEmpty, preferedRegionListOverride: preferredList, - enforceSingleMasterSingleWriteLocation: true); + enforceSingleMasterSingleWriteLocation: true, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); if (isPreferredLocationsEmpty) { - Assert.IsNotNull(this.cache.EffectivePreferredLocations); - Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + if (enableMultipleWriteLocations) + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + } + else + { + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } + } } - + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) { int retryCount = 0; @@ -840,9 +1090,22 @@ await BackoffRetryUtility.ExecuteAsync( || isReadRequest) { // Next request must go to next preferred endpoint - expectedEndpoint = isPreferredLocationsEmpty ? - LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]] - : LocationCacheTests.EndpointByLocation[preferredList[1]]; + // [or] back to first effective preferred region in case empty preferred regions and regional default endpoint + if (isPreferredLocationsEmpty) + { + if (isDefaultEndpointARegionalEndpoint) + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[0]]; + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[this.cache.EffectivePreferredLocations[1]]; + } + } + else + { + expectedEndpoint = LocationCacheTests.EndpointByLocation[preferredList[1]]; + } } else { @@ -871,28 +1134,45 @@ await BackoffRetryUtility.ExecuteAsync( } [DataTestMethod] - [DataRow(true, false, false, true, false, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should retry")] - [DataRow(false, false, false, false, false, DisplayName = "Write request - Single master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(true, true, false, true, false, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should retry")] - [DataRow(false, true, false, true, false, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry")] - [DataRow(true, false, true, true, false, DisplayName = "Read request - Single master - with preferred locations - without partition level failover - should retry")] - [DataRow(false, false, true, false, false, DisplayName = "Write request - Single master - with preferred locations - without partition level failover - should NOT retry")] - [DataRow(true, true, true, true, false, DisplayName = "Read request - Multi master - with preferred locations - without partition level failover - should retry")] - [DataRow(false, true, true, true, false, DisplayName = "Write request - Multi master - with preferred locations - without partition level failover - should retry")] - [DataRow(true, false, false, true, true, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should retry")] - [DataRow(false, false, false, true, true, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should retry")] - [DataRow(true, true, false, true, true, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should retry")] - [DataRow(false, true, false, true, true, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should retry")] - [DataRow(true, false, true, true, true, DisplayName = "Read request - Single master - with preferred locations - with partition level failover - should NOT retry")] - [DataRow(false, false, true, true, true, DisplayName = "Write request - Single master - with preferred locations - with partition level failover - should retry")] - [DataRow(true, true, true, true, true, DisplayName = "Read request - Multi master - with preferred locations - with partition level failover - should retry")] - [DataRow(false, true, true, true, true, DisplayName = "Write request - Multi master - with preferred locations - with partition level failover - should retry")] + [DataRow(true, false, false, true, false, false, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, false, false, false, false, false, DisplayName = "Write request - Single master - no preferred locations - without partition level failover - should NOT retry - global default endpoint")] + [DataRow(true, true, false, true, false, false, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, true, false, true, false, false, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry - global default endpoint")] + [DataRow(true, false, true, true, false, false, DisplayName = "Read request - Single master - with preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, false, true, false, false, false, DisplayName = "Write request - Single master - with preferred locations - without partition level failover - should NOT retry - global default endpoint")] + [DataRow(true, true, true, true, false, false, DisplayName = "Read request - Multi master - with preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(false, true, true, true, false, false, DisplayName = "Write request - Multi master - with preferred locations - without partition level failover - should retry - global default endpoint")] + [DataRow(true, false, false, true, true, false, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(false, false, false, true, true, false, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, true, false, true, true, false, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(false, true, false, true, true, false, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, false, true, true, true, false, DisplayName = "Read request - Single master - with preferred locations - with partition level failover - should NOT retry - global default endpoint")] + [DataRow(false, false, true, true, true, false, DisplayName = "Write request - Single master - with preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, true, true, true, true, false, DisplayName = "Read request - Multi master - with preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(false, true, true, true, true, false, DisplayName = "Write request - Multi master - with preferred locations - with partition level failover - should retry - global default endpoint")] + [DataRow(true, false, false, false, false, true, DisplayName = "Read request - Single master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, false, false, false, false, true, DisplayName = "Write request - Single master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, true, false, false, false, true, DisplayName = "Read request - Multi master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, true, false, false, false, true, DisplayName = "Write request - Multi master - no preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, false, true, true, false, true, DisplayName = "Read request - Single master - with preferred locations - without partition level failover - should retry - regional default endpoint")] + [DataRow(false, false, true, false, false, true, DisplayName = "Write request - Single master - with preferred locations - without partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, true, true, true, false, true, DisplayName = "Read request - Multi master - with preferred locations - without partition level failover - should retry - regional default endpoint")] + [DataRow(false, true, true, true, false, true, DisplayName = "Write request - Multi master - with preferred locations - without partition level failover - should retry - regional default endpoint")] + [DataRow(true, false, false, false, true, true, DisplayName = "Read request - Single master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, false, false, false, true, true, DisplayName = "Write request - Single master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, true, false, false, true, true, DisplayName = "Read request - Multi master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, true, false, false, true, true, DisplayName = "Write request - Multi master - no preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(true, false, true, true, true, true, DisplayName = "Read request - Single master - with preferred locations - with partition level failover - should NOT retry - regional default endpoint")] + [DataRow(false, false, true, true, true, true, DisplayName = "Write request - Single master - with preferred locations - with partition level failover - should retry - regional default endpoint")] + [DataRow(true, true, true, true, true, true, DisplayName = "Read request - Multi master - with preferred locations - with partition level failover - should retry - regional default endpoint")] + [DataRow(false, true, true, true, true, true, DisplayName = "Write request - Multi master - with preferred locations - with partition level failover - should retry - regional default endpoint")] public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, bool shouldHaveRetried, - bool enablePartitionLevelFailover) + bool enablePartitionLevelFailover, + bool isDefaultEndpointARegionalEndpoint) { const bool enableEndpointDiscovery = true; @@ -907,7 +1187,8 @@ public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( isPreferredLocationsListEmpty: !usesPreferredLocations, enablePartitionLevelFailover: enablePartitionLevelFailover, preferedRegionListOverride: preferredList, - enforceSingleMasterSingleWriteLocation: true); + enforceSingleMasterSingleWriteLocation: true, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); @@ -915,8 +1196,17 @@ public async Task ClientRetryPolicy_ValidateRetryOnServiceUnavailable( if (!usesPreferredLocations) { - Assert.IsNotNull(this.cache.EffectivePreferredLocations); - Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + if (isDefaultEndpointARegionalEndpoint) + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(1, this.cache.EffectivePreferredLocations.Count); + Assert.AreEqual("location1", this.cache.EffectivePreferredLocations[0]); + } + else + { + Assert.IsNotNull(this.cache.EffectivePreferredLocations); + Assert.AreEqual(4, this.cache.EffectivePreferredLocations.Count); + } } From 0fb42025752400660eb7a871a8ede584d53cb047 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 16 Sep 2024 13:23:46 -0400 Subject: [PATCH 12/16] Fix LocationCacheTests.cs --- .../src/GatewayAccountReader.cs | 1 - .../src/Routing/LocationCache.cs | 21 ++-- .../GlobalEndpointManagerTest.cs | 5 +- .../LocationCacheTests.cs | 108 ++++++++++++------ 4 files changed, 91 insertions(+), 44 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index 20c795ea12..6d4b6e94f6 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos { using System; - using System.Collections.Generic; using System.Net.Http; using System.Threading; using System.Threading.Tasks; diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index 7c6a2ae830..c5875e9179 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -253,14 +253,6 @@ public ReadOnlyDictionary GetAvailableReadEndpointsByLocation() return this.locationInfo.AvailableReadEndpointByLocation; } - /// - /// Gets account-level write locations. - /// - public ReadOnlyCollection GetAvailableWriteLocations() - { - return this.locationInfo.AvailableWriteLocations; - } - public Uri GetHubUri() { DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo; @@ -269,10 +261,21 @@ public Uri GetHubUri() return locationEndpointToRoute; } + /// + /// Gets account-level read locations. + /// public ReadOnlyCollection GetAvailableReadLocations() { return this.locationInfo.AvailableReadLocations; } + + /// + /// Gets account-level write locations. + /// + public ReadOnlyCollection GetAvailableWriteLocations() + { + return this.locationInfo.AvailableWriteLocations; + } /// /// Resolves request to service endpoint. @@ -525,7 +528,7 @@ public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not available for read.", mostPreferredLocation); return true; } - } + } else { DefaultTrace.TraceInformation("ShouldRefreshEndpoints = true since most preferred location {0} is not in available read locations.", mostPreferredLocation); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index a3ae4b8d25..dda505e731 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -428,7 +428,10 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() } /// - /// Tests for + /// Test to validate for a client that has been warmed up with account-level regions, any subsequent + /// DatabaseAccount refresh calls should go through the effective preferred regions / account-level read regions + /// if the DatabaseAccount refresh call to the global / default endpoint failed with HttpRequestException (timeout also but not possible to inject + /// w/o adding a refresh method just for this test) /// [TestMethod] public async Task GetDatabaseAccountFromEffectiveRegionalEndpointTestAsync() diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs index df9251fb76..d58770d33d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs @@ -967,23 +967,33 @@ await BackoffRetryUtility.ExecuteAsync( } [TestMethod] - [DataRow(true, true, true)] - [DataRow(true, false, false)] - [DataRow(true, false, true)] - [DataRow(true, true, false)] - [DataRow(false, false, false)] - [DataRow(false, true, true)] - [DataRow(false, true, false)] - [DataRow(false, true, true)] + [DataRow(true, true, true, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, false, false, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, false, true, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, true, false, false, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, false, false, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, true, true, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, true, false, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(false, true, true, false, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsGlobalEndpoint")] + [DataRow(true, true, true, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(true, false, false, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(true, false, true, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(true, true, false, true, DisplayName = "MultipleWriteEndpointsEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, false, false, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryNotEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, true, true, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, true, false, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListNotEmpty | DefaultEndpointIsRegionalEndpoint")] + [DataRow(false, true, true, true, DisplayName = "MultipleWriteEndpointsNotEnabled | EndpointDiscoveryEnabled | PreferredLocationListEmpty | DefaultEndpointIsRegionalEndpoint")] public async Task ValidateAsync( bool useMultipleWriteEndpoints, bool endpointDiscoveryEnabled, - bool isPreferredListEmpty) + bool isPreferredListEmpty, + bool isDefaultEndpointARegionalEndpoint) { await this.ValidateLocationCacheAsync( useMultipleWriteEndpoints, endpointDiscoveryEnabled, - isPreferredListEmpty); + isPreferredListEmpty, + isDefaultEndpointARegionalEndpoint); } [TestMethod] @@ -1514,8 +1524,7 @@ private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool us private static AccountProperties CreateDatabaseAccount( bool useMultipleWriteLocations, - bool enforceSingleMasterSingleWriteLocation, - bool isExcludeRegionsTest = false) + bool enforceSingleMasterSingleWriteLocation) { Collection writeLocations = new Collection() { @@ -1565,8 +1574,7 @@ private GlobalEndpointManager Initialize( { this.databaseAccount = LocationCacheTests.CreateDatabaseAccount( useMultipleWriteLocations, - enforceSingleMasterSingleWriteLocation, - isExcludeRegionsTest); + enforceSingleMasterSingleWriteLocation); if (isPreferredLocationsListEmpty) { @@ -1619,9 +1627,9 @@ private GlobalEndpointManager Initialize( private async Task ValidateLocationCacheAsync( bool useMultipleWriteLocations, bool endpointDiscoveryEnabled, - bool isPreferredListEmpty) + bool isPreferredListEmpty, + bool isDefaultEndpointARegionalEndpoint) { - // hardcoded to represent - (location1, location2, location3) as the write regions (with and without preferred regions set) int maxWriteLocationIndex = 3; @@ -1629,14 +1637,21 @@ private async Task ValidateLocationCacheAsync( // as the read regions (with preferred regions set) int maxReadLocationIndex = isPreferredListEmpty ? 4 : 3; + if (isPreferredListEmpty && isDefaultEndpointARegionalEndpoint) + { + maxWriteLocationIndex = 1; + maxReadLocationIndex = 1; + } + for (int writeLocationIndex = 0; writeLocationIndex < maxWriteLocationIndex; writeLocationIndex++) { for (int readLocationIndex = 0; readLocationIndex < maxReadLocationIndex; readLocationIndex++) { using GlobalEndpointManager endpointManager = this.Initialize( - useMultipleWriteLocations, - endpointDiscoveryEnabled, - isPreferredListEmpty); + useMultipleWriteLocations: useMultipleWriteLocations, + enableEndpointDiscovery: endpointDiscoveryEnabled, + isPreferredLocationsListEmpty: isPreferredListEmpty, + isDefaultEndpointARegionalEndpoint: isDefaultEndpointARegionalEndpoint); ReadOnlyCollection currentWriteEndpoints = this.cache.WriteEndpoints; ReadOnlyCollection currentReadEndpoints = this.cache.ReadEndpoints; @@ -1719,7 +1734,9 @@ private async Task ValidateLocationCacheAsync( useMultipleWriteLocations, endpointDiscoveryEnabled, preferredAvailableWriteEndpoints, - preferredAvailableReadEndpoints); + preferredAvailableReadEndpoints, + isPreferredListEmpty, + isDefaultEndpointARegionalEndpoint); // wait for TTL on unavailability info string expirationTime = System.Configuration.ConfigurationManager.AppSettings["UnavailableLocationsExpirationTimeInSeconds"]; @@ -1843,15 +1860,17 @@ private void ValidateRequestEndpointResolution( bool useMultipleWriteLocations, bool endpointDiscoveryEnabled, Uri[] availableWriteEndpoints, - Uri[] availableReadEndpoints) + Uri[] availableReadEndpoints, + bool isPreferredLocationsListEmpty, + bool isDefaultEndpointARegionalEndpoint) { Uri firstAvailableWriteEndpoint; Uri secondAvailableWriteEndpoint; if (!endpointDiscoveryEnabled) { - firstAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; - secondAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; + firstAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; + secondAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; } else if (!useMultipleWriteLocations) { @@ -1860,28 +1879,45 @@ private void ValidateRequestEndpointResolution( } else if (availableWriteEndpoints.Length > 1) { - firstAvailableWriteEndpoint = availableWriteEndpoints[0]; - secondAvailableWriteEndpoint = availableWriteEndpoints[1]; + + if (isDefaultEndpointARegionalEndpoint && isPreferredLocationsListEmpty) + { + firstAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + secondAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + } + else + { + firstAvailableWriteEndpoint = availableWriteEndpoints[0]; + secondAvailableWriteEndpoint = availableWriteEndpoints[1]; + } } else if (availableWriteEndpoints.Length > 0) { - firstAvailableWriteEndpoint = availableWriteEndpoints[0]; - secondAvailableWriteEndpoint = - this.databaseAccount.WriteLocationsInternal[0].Endpoint != firstAvailableWriteEndpoint.ToString() ? - new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint) : - new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); + if (isDefaultEndpointARegionalEndpoint && isPreferredLocationsListEmpty) + { + firstAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + secondAvailableWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + } + else + { + firstAvailableWriteEndpoint = availableWriteEndpoints[0]; + secondAvailableWriteEndpoint = + this.databaseAccount.WriteLocationsInternal[0].Endpoint != firstAvailableWriteEndpoint.ToString() ? + new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint) : + new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); + } } else { - firstAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; - secondAvailableWriteEndpoint = LocationCacheTests.DefaultEndpoint; + firstAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; + secondAvailableWriteEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; } Uri firstAvailableReadEndpoint; if (!endpointDiscoveryEnabled) { - firstAvailableReadEndpoint = LocationCacheTests.DefaultEndpoint; + firstAvailableReadEndpoint = isDefaultEndpointARegionalEndpoint ? LocationCacheTests.DefaultRegionalEndpoint : LocationCacheTests.DefaultEndpoint; } else if (availableReadEndpoints.Length > 0) { @@ -1900,6 +1936,12 @@ private void ValidateRequestEndpointResolution( LocationCacheTests.DefaultEndpoint : new Uri(this.databaseAccount.WriteLocationsInternal[1].Endpoint); + if (isDefaultEndpointARegionalEndpoint && !endpointDiscoveryEnabled) + { + firstWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + secondWriteEndpoint = LocationCacheTests.DefaultRegionalEndpoint; + } + // If current write endpoint is unavailable, write endpoints order doesn't change // All write requests flip-flop between current write and alternate write endpoint ReadOnlyCollection writeEndpoints = this.cache.WriteEndpoints; From b9860a7b169747fa899260fc7cb408f9ab74b2f6 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 16 Sep 2024 16:53:57 -0400 Subject: [PATCH 13/16] Wiring ExcludeRegions for ReadMany. --- Microsoft.Azure.Cosmos/src/ReadManyRequestOptions.cs | 3 ++- .../src/Resource/Container/ContainerInlineCore.cs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ReadManyRequestOptions.cs b/Microsoft.Azure.Cosmos/src/ReadManyRequestOptions.cs index 842faaca76..7535d1c6bd 100644 --- a/Microsoft.Azure.Cosmos/src/ReadManyRequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/ReadManyRequestOptions.cs @@ -68,7 +68,8 @@ internal QueryRequestOptions ConvertToQueryRequestOptions() IfMatchEtag = this.IfMatchEtag, IfNoneMatchEtag = this.IfNoneMatchEtag, Properties = this.Properties, - AddRequestHeaders = this.AddRequestHeaders + AddRequestHeaders = this.AddRequestHeaders, + ExcludeRegions = this.ExcludeRegions }; } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInlineCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInlineCore.cs index f3153a33cf..170d38a975 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInlineCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInlineCore.cs @@ -428,7 +428,7 @@ public override Task ReadManyItemsStreamAsync( containerName: this.Id, databaseName: this.Database.Id, operationType: Documents.OperationType.Read, - requestOptions: null, + requestOptions: readManyRequestOptions, task: (trace) => base.ReadManyItemsStreamAsync(items, trace, readManyRequestOptions, cancellationToken), openTelemetry: (response) => new OpenTelemetryResponse(responseMessage: response)); } @@ -443,7 +443,7 @@ public override Task> ReadManyItemsAsync( containerName: this.Id, databaseName: this.Database.Id, operationType: Documents.OperationType.Read, - requestOptions: null, + requestOptions: readManyRequestOptions, task: (trace) => base.ReadManyItemsAsync(items, trace, readManyRequestOptions, cancellationToken), openTelemetry: (response) => new OpenTelemetryResponse(responseMessage: response)); } From 39a4c8c7b822f147cae282f2b41cb0b8bc5127c0 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 18 Sep 2024 19:46:50 -0400 Subject: [PATCH 14/16] Attempt at fixing line separators. --- Microsoft.Azure.Cosmos.sln | 16 ++++++++++++++-- global.json | 7 +++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 global.json diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index d412905195..f319d88bcd 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29123.88 +# Visual Studio Version 17 +VisualStudioVersion = 17.11.35303.130 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -164,6 +164,18 @@ Global {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|Any CPU.Build.0 = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.ActiveCfg = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/global.json b/global.json new file mode 100644 index 0000000000..9e5e1fd1dc --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "6.0.0", + "rollForward": "latestMajor", + "allowPrerelease": true + } +} \ No newline at end of file From d1be991dd8977a11f6d7412792bd405203a5f753 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 18 Sep 2024 20:11:43 -0400 Subject: [PATCH 15/16] Delete global.json. --- global.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 global.json diff --git a/global.json b/global.json deleted file mode 100644 index 9e5e1fd1dc..0000000000 --- a/global.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sdk": { - "version": "6.0.0", - "rollForward": "latestMajor", - "allowPrerelease": true - } -} \ No newline at end of file From bcfc960a0f9dacfb790e9b10f70d56a42bd96643 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 18 Sep 2024 20:36:10 -0400 Subject: [PATCH 16/16] Fix PR. --- Microsoft.Azure.Cosmos.sln | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index f319d88bcd..d412905195 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.11.35303.130 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29123.88 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -164,18 +164,6 @@ Global {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|Any CPU.Build.0 = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.ActiveCfg = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.Build.0 = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.ActiveCfg = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.Build.0 = Debug|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU - {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE