bounded stalenness test with fix

Azure · Jul 22, 2024 · f430299 · f430299
1 parent cbfde3a
commit f430299
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 5 deletions.
diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/FaultInjection.csproj b/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/FaultInjection.csproj
@@ -26,7 +26,6 @@
 	</ItemGroup>
 
 	<ItemGroup Condition=" '$(ProjectRef)' != 'True' ">
-		<PackageReference Include="Microsoft.Azure.Cosmos.Direct" Version="[$(DirectVersion)]" PrivateAssets="All" />
 		<PackageReference Include="Microsoft.HybridRow" Version="[$(HybridRowVersion)]" PrivateAssets="All" />
 		<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
 	</ItemGroup>

diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/tests/FaultInjectionTests.csproj b/Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/tests/FaultInjectionTests.csproj
@@ -17,7 +17,6 @@
   </PropertyGroup>
 
   <ItemGroup Condition=" '$(ProjectRef)' != 'True' ">
-    <PackageReference Include="Microsoft.Azure.Cosmos.Direct" Version="[$(DirectVersion)]" PrivateAssets="All" />
     <PackageReference Include="Microsoft.HybridRow" Version="[$(HybridRowVersion)]" PrivateAssets="All" />
   </ItemGroup>
 

diff --git a/Microsoft.Azure.Cosmos/src/RMResources.Designer.cs b/Microsoft.Azure.Cosmos/src/RMResources.Designer.cs
diff --git a/Microsoft.Azure.Cosmos/src/direct/QuorumReader.cs b/Microsoft.Azure.Cosmos/src/direct/QuorumReader.cs
@@ -795,7 +795,14 @@ private bool IsQuorumMet(
             selectedResponse = validReadResponses.Where(s => (s.Target.LSN == maxLsn) && (s.Target.StatusCode < StatusCodes.StartingErrorCode)).FirstOrDefault();
             if (selectedResponse == null)
             {
-                selectedResponse = validReadResponses.First(s => s.Target.LSN == maxLsn);
+                try
+                {
+                    selectedResponse = validReadResponses.First(s => s.Target.LSN == maxLsn);
+                }
+                catch
+                {
+                    selectedResponse = validReadResponses.First();
+                }
             }
 
             readLsn = selectedResponse.Target.ItemLSN == -1 ? 

diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs
@@ -33,6 +33,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
     using System.Reflection;
     using System.Text.RegularExpressions;
     using Microsoft.Azure.Cosmos.Diagnostics;
+    using Microsoft.Azure.Cosmos.FaultInjection;
 
     [TestClass]
     public class CosmosItemTests : BaseCosmosClientHelper
@@ -3320,6 +3321,79 @@ await CosmosItemTests.GivenItemAsyncWhenMissingMemberHandlingIsErrorThenExpectsC
                     cancellationToken: cancellationToken));
         }
 
+        [TestMethod]
+        public async Task TooManyReadTest()
+        {
+            string connectionString = ConfigurationManager.GetEnvironmentVariable<string>("COSMOSDB_MULTI_REGION", null);
+
+            if (string.IsNullOrEmpty(connectionString))
+            {
+                Assert.Fail("Set environment variable COSMOSDB_MULTI_REGION to run the tests");
+            }
+
+            FaultInjectionCondition readConditon = new FaultInjectionConditionBuilder()
+                .WithOperationType(FaultInjectionOperationType.ReadItem)
+                .Build();
+            IFaultInjectionResult tooManyRequestsResult = FaultInjectionResultBuilder
+                .GetResultBuilder(FaultInjectionServerErrorType.TooManyRequests)
+                .Build();
+            FaultInjectionRule rule = new FaultInjectionRuleBuilder(
+                id: "tooMany",
+                condition: readConditon,
+                result: tooManyRequestsResult)
+                .WithDuration(TimeSpan.FromMinutes(90))
+                .Build();
+
+            List<FaultInjectionRule> rules = new List<FaultInjectionRule>() { rule };
+            FaultInjector faultInjector = new FaultInjector(rules);
+
+            rule.Disable();
+
+
+            CosmosClientOptions clientOptions = new CosmosClientOptions()
+            {
+                ConnectionMode = ConnectionMode.Direct,
+                ConsistencyLevel = Cosmos.ConsistencyLevel.Strong,
+            };
+
+            CosmosClient faultInjectionClient = new CosmosClient(
+                connectionString: connectionString,
+                clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions));
+
+            Cosmos.Database db = await faultInjectionClient.CreateDatabaseIfNotExistsAsync("LoadTest");
+            Container container = await db.CreateContainerIfNotExistsAsync("LoadContainer", "/pk");
+
+            dynamic item = new
+            {
+                id = "testId",
+                pk = "pk",
+            };
+
+            await container.CreateItemAsync<dynamic>(item);
+
+            rule.Enable();
+
+            try
+            {
+                ItemResponse<dynamic> ir = await container.ReadItemAsync<dynamic>(
+                        "testId",
+                        new Cosmos.PartitionKey("pk"),
+                        new ItemRequestOptions()
+                        {
+                            ExcludeRegions = new List<string>() { "West US" }
+                        });
+
+
+                Assert.AreEqual(HttpStatusCode.OK, ir.StatusCode);
+            }
+            finally
+            {
+                await db.DeleteAsync();
+                faultInjectionClient.Dispose();
+            }
+
+        }
+
         private static async Task GivenItemStreamAsyncWhenMissingMemberHandlingIsErrorThenExpectsCosmosExceptionTestAsync(
             Func<Container, string, string, CancellationToken, Task<ResponseMessage>> itemStreamAsync)
         {

diff --git a/...os/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj b/...os/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj
@@ -19,7 +19,6 @@
   </ItemGroup>
 
   <ItemGroup Condition=" '$(ProjectRef)' != 'True' ">
-    <PackageReference Include="Microsoft.Azure.Cosmos.Direct" Version="[$(DirectVersion)]" PrivateAssets="All" />
     <PackageReference Include="Microsoft.HybridRow" Version="[$(HybridRowVersion)]" PrivateAssets="All" />
   </ItemGroup>
 
@@ -361,6 +360,9 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\Microsoft.Azure.Cosmos.Samples\Tools\FaultInjection\src\FaultInjection.csproj" />
+  </ItemGroup>
 
   <PropertyGroup>
     <SignAssembly>true</SignAssembly>