[Internal] Query: Adds new header SupportedSerializationFormats (#3911)

* Binary Serilaization Response test * Added new header SupportedSerializationFormats * Modified existing use of CosmosSerializationFormatOptions * Modified tests and removed unused code * Addressed comments * Added more negative cases * Revert changes * Added spaces * Addressed comments * Addressed comments * Removed SupportedSerializationFormats from Headers file * Removed unused JsonSerilazationFormats option * Addressed comments * Addressed comments * Addressed comments * Addressed comments * Added new enum TransportSerializationFormat * Added new enum TransportSerializationFormat * Addressed comments * Removed unused parameter * Addressed comments * updating API * remove tests * Text fixes * fix typo * remove TransportSerializationFormat header * text reverts * revert * test update * PR comments * remove test owner headers HeadersValidationTests.cs * PR comments - remove unsupported tests and scope client --------- Co-authored-by: Heet <[email protected]> Co-authored-by: neildsh <[email protected]> Co-authored-by: Matias Quaranta <[email protected]>
Azure · Jun 22, 2023 · 92319a0 · 92319a0
1 parent b1d3134
commit 92319a0
Show file tree

Hide file tree

Showing 19 changed files with 383 additions and 127 deletions.
diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeed/Pagination/ChangeFeedPaginationOptions.cs b/Microsoft.Azure.Cosmos/src/ChangeFeed/Pagination/ChangeFeedPaginationOptions.cs
@@ -33,16 +33,19 @@ public ChangeFeedPaginationOptions(
             JsonSerializationFormat? jsonSerializationFormat = null,
             Dictionary<string, string> additionalHeaders = null,
             ChangeFeedQuerySpec changeFeedQuerySpec = null)
-            : base(pageSizeHint, jsonSerializationFormat, additionalHeaders)
+            : base(pageSizeHint, additionalHeaders)
         {
             this.Mode = mode ?? throw new ArgumentNullException(nameof(mode));
             this.ChangeFeedQuerySpec = changeFeedQuerySpec;
+            this.JsonSerializationFormat = jsonSerializationFormat;
         }
 
         public ChangeFeedMode Mode { get; }
 
         public ChangeFeedQuerySpec ChangeFeedQuerySpec { get; }
 
+        public JsonSerializationFormat? JsonSerializationFormat { get; }
+
         protected override ImmutableHashSet<string> BannedAdditionalHeaders => BannedHeaders;
     }
 }
diff --git a/Microsoft.Azure.Cosmos/src/FeedOptions.cs b/Microsoft.Azure.Cosmos/src/FeedOptions.cs
@@ -24,7 +24,7 @@ public FeedOptions()
         {
         }
 
-        internal FeedOptions(FeedOptions options)
+        public FeedOptions(FeedOptions options)
         {
             if (options == null)
             {
@@ -65,10 +65,10 @@ internal FeedOptions(FeedOptions options)
             this.EndId = options.EndId;
             this.StartEpk = options.StartEpk;
             this.EndEpk = options.EndEpk;
-            this.ContentSerializationFormat = options.ContentSerializationFormat;
             this.EnableGroupBy = options.EnableGroupBy;
             this.MergeStaticId = options.MergeStaticId;
             this.Properties = options.Properties;
+            this.SupportedSerializationFormats = options.SupportedSerializationFormats;
         }
 
         /// <summary>
@@ -282,7 +282,7 @@ internal FeedOptions(FeedOptions options)
         /// <value>
         /// Option is true if query tracing is enabled; otherwise, false.
         /// </value>
-        internal bool? EmitVerboseTracesInQuery { get; set; }
+        public bool? EmitVerboseTracesInQuery { get; set; }
 
         /// <summary>
         /// Gets or sets the schema rid which could be used to filter the document feed response
@@ -292,7 +292,7 @@ internal FeedOptions(FeedOptions options)
         /// By default, it is <c>null</c> which means no filtering will be applied.
         /// Otherwise, it must be a valid resource id of Schema resource.
         /// </value>
-        internal string FilterBySchemaResourceId { get; set; }
+        public string FilterBySchemaResourceId { get; set; }
 
         /// <summary>
         ///  Gets or sets the <see cref="PopulateQueryMetrics"/> request option for document query requests in the Azure Cosmos DB service.
@@ -359,65 +359,62 @@ internal FeedOptions(FeedOptions options)
         /// <summary>
         /// Gets or sets the flag that tells the backend to forces the query to perform a scan (at a request level).
         /// </summary>
-        internal bool ForceQueryScan { get; set; }
+        public bool ForceQueryScan { get; set; }
 
         /// <summary>
         /// Gets or sets the EnumerationDirection
         /// To be used along with Read feed operation
         /// </summary>
-        internal EnumerationDirection? EnumerationDirection { get; set; }
+        public EnumerationDirection? EnumerationDirection { get; set; }
 
         /// <summary>
         /// Gets or sets the ReadFeedKeyType
         /// To be used along with Read feed operation
         /// </summary>
-        internal ReadFeedKeyType? ReadFeedKeyType { get; set; }
+        public ReadFeedKeyType? ReadFeedKeyType { get; set; }
 
         /// <summary>
         /// Gets or sets the StartId
         /// To be used along with Read feed operation
         /// </summary>
-        internal string StartId { get; set; }
+        public string StartId { get; set; }
 
         /// <summary>
         /// Gets or sets the EndId
         /// To be used along with Read feed operation
         /// </summary>
-        internal string EndId { get; set; }
+        public string EndId { get; set; }
 
         /// <summary>
         /// Gets or sets the StartEpk
         /// To be used along with Read feed operation
         /// </summary>
-        internal string StartEpk { get; set; }
+        public string StartEpk { get; set; }
 
         /// <summary>
         /// Gets or sets the EndEpk
         /// To be used along with Read feed operation
         /// </summary>
-        internal string EndEpk { get; set; }
+        public string EndEpk { get; set; }
 
         /// <summary>
-        /// Gets or sets the ContentSerializationFormat for the feed (query/read feed) operation in the Azure Cosmos DB service.
+        /// Gets or sets the SupportedSerializationFormats for the query response from the service.
         /// </summary>
-        /// <remarks>
-        /// If the document is stored in a different serialization format then the one requested, then there will be a rewrite over the wire, but the source document will be untouched.
-        /// </remarks>
-        internal ContentSerializationFormat? ContentSerializationFormat { get; set; }
+        public SupportedSerializationFormats? SupportedSerializationFormats { get; set; }
 
-        internal bool EnableGroupBy { get; set; }
+        public bool EnableGroupBy { get; set; }
 
         /// <summary>
         /// Gets or sets the MergeStaticId.
         /// To be used along with Read feed operation when Static Column merge is desired.
         /// </summary>
-        internal string MergeStaticId { get; set; }
+        public string MergeStaticId { get; set; }
 
         /// <summary>
         /// Gets or sets the custom serialization options for query
         /// </summary>
-        internal CosmosSerializationFormatOptions CosmosSerializationFormatOptions { get; set; }
+        public CosmosSerializationFormatOptions CosmosSerializationFormatOptions { get; set; }
 
-        internal IDictionary<string, object> Properties { get; set; }
+        public IDictionary<string, object> Properties { get; set; }
     }
 }
diff --git a/Microsoft.Azure.Cosmos/src/Headers/CosmosMessageHeadersInternal.cs b/Microsoft.Azure.Cosmos/src/Headers/CosmosMessageHeadersInternal.cs
@@ -159,6 +159,12 @@ public virtual string ContentSerializationFormat
             set => this.SetProperty(HttpConstants.HttpHeaders.ContentSerializationFormat, value);
         }
 
+        public virtual string SupportedSerializationFormats
+        {
+            get => this.GetValueOrDefault(HttpConstants.HttpHeaders.SupportedSerializationFormats);
+            set => this.SetProperty(HttpConstants.HttpHeaders.SupportedSerializationFormats, value);
+        }
+
         public virtual string ReadFeedKeyType
         {
             get => this.GetValueOrDefault(HttpConstants.HttpHeaders.ReadFeedKeyType);

diff --git a/Microsoft.Azure.Cosmos/src/Headers/StoreRequestHeaders.cs b/Microsoft.Azure.Cosmos/src/Headers/StoreRequestHeaders.cs
@@ -76,6 +76,12 @@ public override string ContentSerializationFormat
             set => this.requestNameValueCollection.ContentSerializationFormat = value;
         }
 
+        public override string SupportedSerializationFormats
+        {
+            get => this.requestNameValueCollection.SupportedSerializationFormats;
+            set => this.requestNameValueCollection.SupportedSerializationFormats = value;
+        }
+
         public override string ReadFeedKeyType
         {
             get => this.requestNameValueCollection.ReadFeedKeyType;

diff --git a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs
@@ -192,11 +192,6 @@ public async Task<TryCatch<ReadFeedPage>> MonadicReadFeedAsync(
                         request.Headers.ContinuationToken = ((CosmosString)readFeedContinuationState.ContinuationToken).Value;
                     }
 
-                    if (readFeedPaginationOptions.JsonSerializationFormat.HasValue)
-                    {
-                        request.Headers.ContentSerializationFormat = readFeedPaginationOptions.JsonSerializationFormat.Value.ToContentSerializationFormatString();
-                    }
-
                     foreach (KeyValuePair<string, string> kvp in readFeedPaginationOptions.AdditionalHeaders)
                     {
                         request.Headers[kvp.Key] = kvp.Value;

diff --git a/Microsoft.Azure.Cosmos/src/Pagination/PaginationOptions.cs b/Microsoft.Azure.Cosmos/src/Pagination/PaginationOptions.cs
@@ -23,11 +23,9 @@ internal abstract class PaginationOptions
 
         protected PaginationOptions(
             int? pageSizeLimit = null,
-            JsonSerializationFormat? jsonSerializationFormat = null,
             Dictionary<string, string> additionalHeaders = null)
         {
             this.PageSizeLimit = pageSizeLimit;
-            this.JsonSerializationFormat = jsonSerializationFormat;
             this.AdditionalHeaders = additionalHeaders != null ? additionalHeaders.ToImmutableDictionary<string, string>() : EmptyDictionary;
 
             foreach (string key in this.AdditionalHeaders.Keys)
@@ -41,8 +39,6 @@ protected PaginationOptions(
 
         public int? PageSizeLimit { get; }
 
-        public JsonSerializationFormat? JsonSerializationFormat { get; }
-
         public ImmutableDictionary<string, string> AdditionalHeaders { get; }
 
         protected abstract ImmutableHashSet<string> BannedAdditionalHeaders { get; }

diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Pagination/QueryPaginationOptions.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Pagination/QueryPaginationOptions.cs
@@ -32,9 +32,8 @@ internal sealed class QueryPaginationOptions : PaginationOptions
         public QueryPaginationOptions(
             int? pageSizeHint = null,
             bool optimisticDirectExecute = false,
-            JsonSerializationFormat? jsonSerializationFormat = null,
             Dictionary<string, string> additionalHeaders = null)
-            : base(pageSizeHint, jsonSerializationFormat, additionalHeaders)
+            : base(pageSizeHint, additionalHeaders)
         {
             this.OptimisticDirectExecute = optimisticDirectExecute;
         }

diff --git a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs
@@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Query
 {
     using System;
     using System.Collections.Generic;
+    using System.ComponentModel;
     using System.Globalization;
     using System.IO;
     using System.Linq;
@@ -31,6 +32,8 @@ namespace Microsoft.Azure.Cosmos.Query
 
     internal abstract class DocumentQueryExecutionContextBase : IDocumentQueryExecutionContext
     {
+        public static readonly string DefaultSupportedSerializationFormats = string.Join(",", SupportedSerializationFormats.JsonText, SupportedSerializationFormats.CosmosBinary);
+
         public readonly struct InitParams
         {
             public IDocumentQueryClient Client { get; }
@@ -326,14 +329,7 @@ await this.Client.EnsureValidOverwriteAsync(
                 requestHeaders.Set(HttpConstants.HttpHeaders.MergeStaticId, this.feedOptions.MergeStaticId);
             }
 
-            if (this.feedOptions.CosmosSerializationFormatOptions != null)
-            {
-                requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.CosmosSerializationFormatOptions.ContentSerializationFormat;
-            }
-            else if (this.feedOptions.ContentSerializationFormat.HasValue)
-            {
-                requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.ContentSerializationFormat.Value.ToString();
-            }
+            requestHeaders[HttpConstants.HttpHeaders.SupportedSerializationFormats] = this.feedOptions.SupportedSerializationFormats?.ToString() ?? DefaultSupportedSerializationFormats;
 
             return requestHeaders;
         }
@@ -688,23 +684,7 @@ private DocumentFeedResponse<CosmosElement> GetFeedResponse(
             {
                 content = memoryStream.ToArray();
             }
-
-            IJsonNavigator jsonNavigator = null;
-
-            // Use the users custom navigator first. If it returns null back try the
-            // internal navigator.
-            if (this.feedOptions.CosmosSerializationFormatOptions != null)
-            {
-                jsonNavigator = this.feedOptions.CosmosSerializationFormatOptions.CreateCustomNavigatorCallback(content);
-                if (jsonNavigator == null)
-                {
-                    throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
-                }
-            }
-            else
-            {
-                jsonNavigator = JsonNavigator.Create(content);
-            }
+            IJsonNavigator jsonNavigator = JsonNavigator.Create(content);
 
             string resourceName = this.GetRootNodeName(documentServiceRequest.ResourceType);
 

diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs
@@ -155,7 +155,6 @@ public override async Task<TryCatch<QueryPage>> ExecuteItemQueryAsync(
                 cancellationToken: cancellationToken);
 
             return CosmosQueryClientCore.GetCosmosElementResponse(
-                requestOptions,
                 resourceType,
                 message,
                 trace);
@@ -297,7 +296,6 @@ public override void ClearSessionTokenCache(string collectionFullName)
         }
 
         private static TryCatch<QueryPage> GetCosmosElementResponse(
-            QueryRequestOptions requestOptions,
             ResourceType resourceType,
             ResponseMessage cosmosResponseMessage,
             ITrace trace)
@@ -336,8 +334,7 @@ private static TryCatch<QueryPage> GetCosmosElementResponse(
                     long responseLengthBytes = memoryStream.Length;
                     CosmosArray documents = CosmosQueryClientCore.ParseElementsFromRestStream(
                         memoryStream,
-                        resourceType,
-                        requestOptions.CosmosSerializationFormatOptions);
+                        resourceType);
 
                     QueryState queryState;
                     if (cosmosResponseMessage.Headers.ContinuationToken != null)
@@ -444,12 +441,10 @@ private Task<PartitionKeyRangeCache> GetRoutingMapProviderAsync()
         /// </summary>
         /// <param name="stream">The memory stream response for the query REST response Azure Cosmos</param>
         /// <param name="resourceType">The resource type</param>
-        /// <param name="cosmosSerializationOptions">The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats</param>
         /// <returns>An array of CosmosElements parsed from the response body.</returns>
         public static CosmosArray ParseElementsFromRestStream(
             Stream stream,
-            ResourceType resourceType,
-            CosmosSerializationFormatOptions cosmosSerializationOptions)
+            ResourceType resourceType)
         {
             if (!(stream is MemoryStream memoryStream))
             {
@@ -478,20 +473,7 @@ public static CosmosArray ParseElementsFromRestStream(
             // You want to create a CosmosElement for each document in "Documents".
 
             ReadOnlyMemory<byte> content = memoryStream.TryGetBuffer(out ArraySegment<byte> buffer) ? buffer : (ReadOnlyMemory<byte>)memoryStream.ToArray();
-            IJsonNavigator jsonNavigator;
-            if (cosmosSerializationOptions != null)
-            {
-                // Use the users custom navigator
-                jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content);
-                if (jsonNavigator == null)
-                {
-                    throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator.");
-                }
-            }
-            else
-            {
-                jsonNavigator = JsonNavigator.Create(content);
-            }
+            IJsonNavigator jsonNavigator = JsonNavigator.Create(content);
 
             string resourceName = resourceType switch
             {

diff --git a/Microsoft.Azure.Cosmos/src/ReadFeed/Pagination/ReadFeedPaginationOptions.cs b/Microsoft.Azure.Cosmos/src/ReadFeed/Pagination/ReadFeedPaginationOptions.cs
@@ -27,9 +27,8 @@ internal sealed class ReadFeedPaginationOptions : PaginationOptions
         public ReadFeedPaginationOptions(
             PaginationDirection? paginationDirection = null,
             int? pageSizeHint = null,
-            JsonSerializationFormat? jsonSerializationFormat = null,
             Dictionary<string, string> additionalHeaders = null)
-            : base(pageSizeHint, jsonSerializationFormat, additionalHeaders)
+            : base(pageSizeHint, additionalHeaders)
         {
             this.Direction = paginationDirection;
         }

diff --git a/Microsoft.Azure.Cosmos/src/ReadFeed/ReadFeedCrossFeedRangeAsyncEnumerator.cs b/Microsoft.Azure.Cosmos/src/ReadFeed/ReadFeedCrossFeedRangeAsyncEnumerator.cs
@@ -45,8 +45,7 @@ public async ValueTask<bool> MoveNextAsync()
 
             CosmosArray documents = CosmosQueryClientCore.ParseElementsFromRestStream(
                 innerReadFeedPage.Page.Content,
-                Documents.ResourceType.Document,
-                cosmosSerializationOptions: null);
+                Documents.ResourceType.Document);
             ReadFeedPage page = new ReadFeedPage(
                 documents,
                 innerReadFeedPage.Page.RequestCharge,

diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs
@@ -5,8 +5,10 @@
 namespace Microsoft.Azure.Cosmos
 {
     using System;
+    using System.ComponentModel;
     using System.Text;
     using Microsoft.Azure.Cosmos.CosmosElements;
+    using Microsoft.Azure.Cosmos.Query;
     using Microsoft.Azure.Cosmos.Query.Core;
     using Microsoft.Azure.Cosmos.Query.Core.Pipeline;
     using Microsoft.Azure.Documents;
@@ -188,6 +190,8 @@ public ConsistencyLevel? ConsistencyLevel
 
         internal CosmosSerializationFormatOptions CosmosSerializationFormatOptions { get; set; }
 
+        internal SupportedSerializationFormats? SupportedSerializationFormats { get; set; }
+
         internal ExecutionEnvironment? ExecutionEnvironment { get; set; }
 
         internal bool? ReturnResultsInDeterministicOrder { get; set; }
@@ -240,11 +244,8 @@ internal override void PopulateRequestOptions(RequestMessage request)
             {
                 request.Headers.Add(HttpConstants.HttpHeaders.ResponseContinuationTokenLimitInKB, this.ResponseContinuationTokenLimitInKb.ToString());
             }
-
-            if (this.CosmosSerializationFormatOptions != null)
-            {
-                request.Headers.CosmosMessageHeaders.ContentSerializationFormat = this.CosmosSerializationFormatOptions.ContentSerializationFormat;
-            }
+
+            request.Headers.CosmosMessageHeaders.SupportedSerializationFormats = this.SupportedSerializationFormats?.ToString() ?? DocumentQueryExecutionContextBase.DefaultSupportedSerializationFormats;
 
             if (this.StartId != null)
             {

diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationFormatOptions.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationFormatOptions.cs
@@ -52,4 +52,4 @@ public CosmosSerializationFormatOptions(
             this.CreateCustomWriterCallback = createCustomWriter;
         }
     }
-}
+}