[Internal] Request Hedging: Refactors HedgeRegion diagnostics field to only show successful region

Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionParallelHedgingAvailabilityStrategy.cs

@@ -4,8 +4,10 @@
 namespace Microsoft.Azure.Cosmos
 {
     using System;
+    using System.Collections.Concurrent;
     using System.Collections.Generic;
     using System.Diagnostics;
+    using System.Dynamic;
     using System.Linq;
     using System.Net;
     using System.Threading;
@@ -22,10 +24,8 @@ namespace Microsoft.Azure.Cosmos
     /// </summary>
     internal class CrossRegionParallelHedgingAvailabilityStrategy : AvailabilityStrategy
     {
-        private const string HedgeRegions = "Hedge Regions";
         private const string HedgeContext = "Hedge Context";
-        private const string HedgeContextOriginalRequest = "Original Request";
-        private const string HedgeContextHedgedRequest = "Hedged Request";
+        private const string ResponseRegion = "Response Region";
 
         /// <summary>
         /// Latency threshold which activates the first region hedging 
@@ -120,9 +120,8 @@ public override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
 
                     List<Task> requestTasks = new List<Task>(hedgeRegions.Count + 1);
 
-                    Task<(bool, ResponseMessage)> primaryRequest = null;
-
-                    ResponseMessage responseMessage = null;
+                    Task<HedgingResponse> primaryRequest = null;
+                    HedgingResponse hedgeResponse = null;
 
                     //Send out hedged requests
                     for (int requestNumber = 0; requestNumber < hedgeRegions.Count; requestNumber++)
@@ -139,14 +138,15 @@ public override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
                                     primaryRequest = this.RequestSenderAndResultCheckAsync(
                                         sender,
                                         request,
+                                        hedgeRegions.ElementAt(requestNumber),
                                         cancellationToken,
                                         cancellationTokenSource);
 
                                     requestTasks.Add(primaryRequest);
                                 }
                                 else
                                 {
-                                    Task<(bool, ResponseMessage)> requestTask = this.CloneAndSendAsync(
+                                    Task<HedgingResponse> requestTask = this.CloneAndSendAsync(
                                     sender: sender,
                                     request: request,
                                     clonedBody: clonedBody,
@@ -176,19 +176,17 @@ public override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
                                     AggregateException innerExceptions = completedTask.Exception.Flatten();
                                 }
 
-                                (bool isNonTransient, responseMessage) = await (Task<(bool, ResponseMessage)>)completedTask;
-                                if (isNonTransient)
+                                hedgeResponse = await (Task<HedgingResponse>)completedTask;
+                                if (hedgeResponse.IsNonTransient)
                                 {
                                     cancellationTokenSource.Cancel();
-                                    ((CosmosTraceDiagnostics)responseMessage.Diagnostics).Value.AddOrUpdateDatum(
-                                        HedgeRegions,
-                                        HedgeRegionsToString(responseMessage.Diagnostics.GetContactedRegions()));
-                                    ((CosmosTraceDiagnostics)responseMessage.Diagnostics).Value.AddOrUpdateDatum(
+                                    ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
                                         HedgeContext,
-                                        object.ReferenceEquals(primaryRequest, completedTask)
-                                            ? HedgeContextOriginalRequest
-                                            : HedgeContextHedgedRequest);
-                                    return responseMessage;
+                                        hedgeRegions.Take(requestNumber + 1));
+                                    ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
+                                        ResponseRegion,
+                                        hedgeResponse.ResponseRegion);
+                                    return hedgeResponse.ResponseMessage;
                                 }
                             }
                         }
@@ -206,19 +204,17 @@ public override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
                             lastException = innerExceptions.InnerExceptions.FirstOrDefault();
                         }
 
-                        (bool isNonTransient, responseMessage) = await (Task<(bool, ResponseMessage)>)completedTask;
-                        if (isNonTransient || requestTasks.Count == 0)
+                        hedgeResponse = await (Task<HedgingResponse>)completedTask;
+                        if (hedgeResponse.IsNonTransient || requestTasks.Count == 0)
                         {
                             cancellationTokenSource.Cancel();
-                            ((CosmosTraceDiagnostics)responseMessage.Diagnostics).Value.AddOrUpdateDatum(
-                                HedgeRegions,
-                                HedgeRegionsToString(responseMessage.Diagnostics.GetContactedRegions()));
-                            ((CosmosTraceDiagnostics)responseMessage.Diagnostics).Value.AddOrUpdateDatum(
+                            ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
                                 HedgeContext,
-                                object.ReferenceEquals(primaryRequest, completedTask)
-                                ? HedgeContextOriginalRequest
-                                : HedgeContextHedgedRequest);
-                            return responseMessage;
+                                hedgeRegions);
+                            ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
+                                        ResponseRegion,
+                                        hedgeResponse.ResponseRegion);
+                            return hedgeResponse.ResponseMessage;
                         }
                     }
 
@@ -227,13 +223,13 @@ public override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
                         throw lastException;
                     }
 
-                    Debug.Assert(responseMessage != null);
-                    return responseMessage;
+                    Debug.Assert(hedgeResponse != null);
+                    return hedgeResponse.ResponseMessage;
                 }
             }
         }
 
-        private async Task<(bool, ResponseMessage)> CloneAndSendAsync(
+        private async Task<HedgingResponse> CloneAndSendAsync(
             Func<RequestMessage, CancellationToken, Task<ResponseMessage>> sender,
             RequestMessage request,
             CloneableStream clonedBody,
@@ -248,20 +244,23 @@ public override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
                 clonedRequest.RequestOptions ??= new RequestOptions();
 
                 List<string> excludeRegions = new List<string>(hedgeRegions);
+                string region = excludeRegions[requestNumber];
                 excludeRegions.RemoveAt(requestNumber);
                 clonedRequest.RequestOptions.ExcludeRegions = excludeRegions;
 
                 return await this.RequestSenderAndResultCheckAsync(
                     sender,
                     clonedRequest,
+                    region,
                     cancellationToken,
                     cancellationTokenSource);
             }
         }
 
-        private async Task<(bool, ResponseMessage)> RequestSenderAndResultCheckAsync(
+        private async Task<HedgingResponse> RequestSenderAndResultCheckAsync(
             Func<RequestMessage, CancellationToken, Task<ResponseMessage>> sender,
             RequestMessage request,
+            string hedgedRegion,
             CancellationToken cancellationToken,
             CancellationTokenSource cancellationTokenSource)
         {
@@ -274,14 +273,15 @@ public override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
                     {
                         cancellationTokenSource.Cancel();
                     }
-                    return (true, response);
+
+                    return new HedgingResponse(true, response, hedgedRegion);
                 }
 
-                return (false, response);
+                return new HedgingResponse(false, response, hedgedRegion);
             }
             catch (OperationCanceledException) when (cancellationTokenSource.IsCancellationRequested)
             {
-                return (false, null);
+                return new HedgingResponse(false, null, hedgedRegion);
             }
             catch (Exception ex)
             {
@@ -315,9 +315,18 @@ private static bool IsFinalResult(int statusCode, int subStatusCode)
             return statusCode == (int)HttpStatusCode.NotFound && subStatusCode == (int)SubStatusCodes.Unknown;
         }
 
-        private static string HedgeRegionsToString(IReadOnlyList<(string, Uri)> hedgeRegions)
+        private class HedgingResponse
         {
-            return string.Join(",", hedgeRegions);
+            public bool IsNonTransient;
+            public ResponseMessage ResponseMessage;
+            public string ResponseRegion;
+
+            public HedgingResponse(bool isNonTransient, ResponseMessage responseMessage, string responseRegion)
+            {
+                this.IsNonTransient = isNonTransient;
+                this.ResponseMessage = responseMessage;
+                this.ResponseRegion = responseRegion;
+            }
         }
     }
 }