From 92319a0528b5e257da71dd6ec0c8bc50375d05c3 Mon Sep 17 00:00:00 2001 From: Maya-Painter <130110800+Maya-Painter@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:19:55 -0700 Subject: [PATCH] [Internal] Query: Adds new header SupportedSerializationFormats (#3911) * Binary Serilaization Response test * Added new header SupportedSerializationFormats * Modified existing use of CosmosSerializationFormatOptions * Modified tests and removed unused code * Addressed comments * Added more negative cases * Revert changes * Added spaces * Addressed comments * Addressed comments * Removed SupportedSerializationFormats from Headers file * Removed unused JsonSerilazationFormats option * Addressed comments * Addressed comments * Addressed comments * Addressed comments * Added new enum TransportSerializationFormat * Added new enum TransportSerializationFormat * Addressed comments * Removed unused parameter * Addressed comments * updating API * remove tests * Text fixes * fix typo * remove TransportSerializationFormat header * text reverts * revert * test update * PR comments * remove test owner headers HeadersValidationTests.cs * PR comments - remove unsupported tests and scope client --------- Co-authored-by: Heet Co-authored-by: neildsh <35383880+neildsh@users.noreply.github.com> Co-authored-by: Matias Quaranta --- .../Pagination/ChangeFeedPaginationOptions.cs | 5 +- Microsoft.Azure.Cosmos/src/FeedOptions.cs | 37 ++-- .../Headers/CosmosMessageHeadersInternal.cs | 6 + .../src/Headers/StoreRequestHeaders.cs | 6 + .../NetworkAttachedDocumentContainer.cs | 5 - .../src/Pagination/PaginationOptions.cs | 4 - .../Pagination/QueryPaginationOptions.cs | 3 +- .../DocumentQueryExecutionContextBase.cs | 30 +-- .../Query/v3Query/CosmosQueryClientCore.cs | 24 +-- .../Pagination/ReadFeedPaginationOptions.cs | 3 +- .../ReadFeedCrossFeedRangeAsyncEnumerator.cs | 3 +- .../src/RequestOptions/QueryRequestOptions.cs | 11 +- .../CosmosSerializationFormatOptions.cs | 2 +- .../HeadersValidationTests.cs | 192 +++++++++++++++++- .../SupportedSerializationFormatsTest.cs | 102 ++++++++++ .../ContentSerializationPerformanceTests.cs | 4 - .../BinaryEncodingOverTheWireTests.cs | 42 ++-- .../OrderByQueryResultTests.cs | 6 +- .../Query/FeedOptionTests.cs | 25 +++ 19 files changed, 383 insertions(+), 127 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeed/Pagination/ChangeFeedPaginationOptions.cs b/Microsoft.Azure.Cosmos/src/ChangeFeed/Pagination/ChangeFeedPaginationOptions.cs index 8579c6de42..85a2895774 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeed/Pagination/ChangeFeedPaginationOptions.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeed/Pagination/ChangeFeedPaginationOptions.cs @@ -33,16 +33,19 @@ public ChangeFeedPaginationOptions( JsonSerializationFormat? jsonSerializationFormat = null, Dictionary additionalHeaders = null, ChangeFeedQuerySpec changeFeedQuerySpec = null) - : base(pageSizeHint, jsonSerializationFormat, additionalHeaders) + : base(pageSizeHint, additionalHeaders) { this.Mode = mode ?? throw new ArgumentNullException(nameof(mode)); this.ChangeFeedQuerySpec = changeFeedQuerySpec; + this.JsonSerializationFormat = jsonSerializationFormat; } public ChangeFeedMode Mode { get; } public ChangeFeedQuerySpec ChangeFeedQuerySpec { get; } + public JsonSerializationFormat? JsonSerializationFormat { get; } + protected override ImmutableHashSet BannedAdditionalHeaders => BannedHeaders; } } diff --git a/Microsoft.Azure.Cosmos/src/FeedOptions.cs b/Microsoft.Azure.Cosmos/src/FeedOptions.cs index 6f4cae7e0d..4d6ce52f41 100644 --- a/Microsoft.Azure.Cosmos/src/FeedOptions.cs +++ b/Microsoft.Azure.Cosmos/src/FeedOptions.cs @@ -24,7 +24,7 @@ public FeedOptions() { } - internal FeedOptions(FeedOptions options) + public FeedOptions(FeedOptions options) { if (options == null) { @@ -65,10 +65,10 @@ internal FeedOptions(FeedOptions options) this.EndId = options.EndId; this.StartEpk = options.StartEpk; this.EndEpk = options.EndEpk; - this.ContentSerializationFormat = options.ContentSerializationFormat; this.EnableGroupBy = options.EnableGroupBy; this.MergeStaticId = options.MergeStaticId; this.Properties = options.Properties; + this.SupportedSerializationFormats = options.SupportedSerializationFormats; } /// @@ -282,7 +282,7 @@ internal FeedOptions(FeedOptions options) /// /// Option is true if query tracing is enabled; otherwise, false. /// - internal bool? EmitVerboseTracesInQuery { get; set; } + public bool? EmitVerboseTracesInQuery { get; set; } /// /// Gets or sets the schema rid which could be used to filter the document feed response @@ -292,7 +292,7 @@ internal FeedOptions(FeedOptions options) /// By default, it is null which means no filtering will be applied. /// Otherwise, it must be a valid resource id of Schema resource. /// - internal string FilterBySchemaResourceId { get; set; } + public string FilterBySchemaResourceId { get; set; } /// /// Gets or sets the request option for document query requests in the Azure Cosmos DB service. @@ -359,65 +359,62 @@ internal FeedOptions(FeedOptions options) /// /// Gets or sets the flag that tells the backend to forces the query to perform a scan (at a request level). /// - internal bool ForceQueryScan { get; set; } + public bool ForceQueryScan { get; set; } /// /// Gets or sets the EnumerationDirection /// To be used along with Read feed operation /// - internal EnumerationDirection? EnumerationDirection { get; set; } + public EnumerationDirection? EnumerationDirection { get; set; } /// /// Gets or sets the ReadFeedKeyType /// To be used along with Read feed operation /// - internal ReadFeedKeyType? ReadFeedKeyType { get; set; } + public ReadFeedKeyType? ReadFeedKeyType { get; set; } /// /// Gets or sets the StartId /// To be used along with Read feed operation /// - internal string StartId { get; set; } + public string StartId { get; set; } /// /// Gets or sets the EndId /// To be used along with Read feed operation /// - internal string EndId { get; set; } + public string EndId { get; set; } /// /// Gets or sets the StartEpk /// To be used along with Read feed operation /// - internal string StartEpk { get; set; } + public string StartEpk { get; set; } /// /// Gets or sets the EndEpk /// To be used along with Read feed operation /// - internal string EndEpk { get; set; } + public string EndEpk { get; set; } /// - /// Gets or sets the ContentSerializationFormat for the feed (query/read feed) operation in the Azure Cosmos DB service. + /// Gets or sets the SupportedSerializationFormats for the query response from the service. /// - /// - /// If the document is stored in a different serialization format then the one requested, then there will be a rewrite over the wire, but the source document will be untouched. - /// - internal ContentSerializationFormat? ContentSerializationFormat { get; set; } + public SupportedSerializationFormats? SupportedSerializationFormats { get; set; } - internal bool EnableGroupBy { get; set; } + public bool EnableGroupBy { get; set; } /// /// Gets or sets the MergeStaticId. /// To be used along with Read feed operation when Static Column merge is desired. /// - internal string MergeStaticId { get; set; } + public string MergeStaticId { get; set; } /// /// Gets or sets the custom serialization options for query /// - internal CosmosSerializationFormatOptions CosmosSerializationFormatOptions { get; set; } + public CosmosSerializationFormatOptions CosmosSerializationFormatOptions { get; set; } - internal IDictionary Properties { get; set; } + public IDictionary Properties { get; set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Headers/CosmosMessageHeadersInternal.cs b/Microsoft.Azure.Cosmos/src/Headers/CosmosMessageHeadersInternal.cs index f12595d832..1cbf9b54e6 100644 --- a/Microsoft.Azure.Cosmos/src/Headers/CosmosMessageHeadersInternal.cs +++ b/Microsoft.Azure.Cosmos/src/Headers/CosmosMessageHeadersInternal.cs @@ -159,6 +159,12 @@ public virtual string ContentSerializationFormat set => this.SetProperty(HttpConstants.HttpHeaders.ContentSerializationFormat, value); } + public virtual string SupportedSerializationFormats + { + get => this.GetValueOrDefault(HttpConstants.HttpHeaders.SupportedSerializationFormats); + set => this.SetProperty(HttpConstants.HttpHeaders.SupportedSerializationFormats, value); + } + public virtual string ReadFeedKeyType { get => this.GetValueOrDefault(HttpConstants.HttpHeaders.ReadFeedKeyType); diff --git a/Microsoft.Azure.Cosmos/src/Headers/StoreRequestHeaders.cs b/Microsoft.Azure.Cosmos/src/Headers/StoreRequestHeaders.cs index 68086095f7..b907363685 100644 --- a/Microsoft.Azure.Cosmos/src/Headers/StoreRequestHeaders.cs +++ b/Microsoft.Azure.Cosmos/src/Headers/StoreRequestHeaders.cs @@ -76,6 +76,12 @@ public override string ContentSerializationFormat set => this.requestNameValueCollection.ContentSerializationFormat = value; } + public override string SupportedSerializationFormats + { + get => this.requestNameValueCollection.SupportedSerializationFormats; + set => this.requestNameValueCollection.SupportedSerializationFormats = value; + } + public override string ReadFeedKeyType { get => this.requestNameValueCollection.ReadFeedKeyType; diff --git a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs index 7d8683c3fe..8420cb0540 100644 --- a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs +++ b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs @@ -192,11 +192,6 @@ public async Task> MonadicReadFeedAsync( request.Headers.ContinuationToken = ((CosmosString)readFeedContinuationState.ContinuationToken).Value; } - if (readFeedPaginationOptions.JsonSerializationFormat.HasValue) - { - request.Headers.ContentSerializationFormat = readFeedPaginationOptions.JsonSerializationFormat.Value.ToContentSerializationFormatString(); - } - foreach (KeyValuePair kvp in readFeedPaginationOptions.AdditionalHeaders) { request.Headers[kvp.Key] = kvp.Value; diff --git a/Microsoft.Azure.Cosmos/src/Pagination/PaginationOptions.cs b/Microsoft.Azure.Cosmos/src/Pagination/PaginationOptions.cs index 5558f2be24..593d761ec0 100644 --- a/Microsoft.Azure.Cosmos/src/Pagination/PaginationOptions.cs +++ b/Microsoft.Azure.Cosmos/src/Pagination/PaginationOptions.cs @@ -23,11 +23,9 @@ internal abstract class PaginationOptions protected PaginationOptions( int? pageSizeLimit = null, - JsonSerializationFormat? jsonSerializationFormat = null, Dictionary additionalHeaders = null) { this.PageSizeLimit = pageSizeLimit; - this.JsonSerializationFormat = jsonSerializationFormat; this.AdditionalHeaders = additionalHeaders != null ? additionalHeaders.ToImmutableDictionary() : EmptyDictionary; foreach (string key in this.AdditionalHeaders.Keys) @@ -41,8 +39,6 @@ protected PaginationOptions( public int? PageSizeLimit { get; } - public JsonSerializationFormat? JsonSerializationFormat { get; } - public ImmutableDictionary AdditionalHeaders { get; } protected abstract ImmutableHashSet BannedAdditionalHeaders { get; } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Pagination/QueryPaginationOptions.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Pagination/QueryPaginationOptions.cs index c285135feb..c985158828 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Pagination/QueryPaginationOptions.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Pagination/QueryPaginationOptions.cs @@ -32,9 +32,8 @@ internal sealed class QueryPaginationOptions : PaginationOptions public QueryPaginationOptions( int? pageSizeHint = null, bool optimisticDirectExecute = false, - JsonSerializationFormat? jsonSerializationFormat = null, Dictionary additionalHeaders = null) - : base(pageSizeHint, jsonSerializationFormat, additionalHeaders) + : base(pageSizeHint, additionalHeaders) { this.OptimisticDirectExecute = optimisticDirectExecute; } diff --git a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs index ec4e6d9185..053c951f6c 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Query { using System; using System.Collections.Generic; + using System.ComponentModel; using System.Globalization; using System.IO; using System.Linq; @@ -31,6 +32,8 @@ namespace Microsoft.Azure.Cosmos.Query internal abstract class DocumentQueryExecutionContextBase : IDocumentQueryExecutionContext { + public static readonly string DefaultSupportedSerializationFormats = string.Join(",", SupportedSerializationFormats.JsonText, SupportedSerializationFormats.CosmosBinary); + public readonly struct InitParams { public IDocumentQueryClient Client { get; } @@ -326,14 +329,7 @@ await this.Client.EnsureValidOverwriteAsync( requestHeaders.Set(HttpConstants.HttpHeaders.MergeStaticId, this.feedOptions.MergeStaticId); } - if (this.feedOptions.CosmosSerializationFormatOptions != null) - { - requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.CosmosSerializationFormatOptions.ContentSerializationFormat; - } - else if (this.feedOptions.ContentSerializationFormat.HasValue) - { - requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.ContentSerializationFormat.Value.ToString(); - } + requestHeaders[HttpConstants.HttpHeaders.SupportedSerializationFormats] = this.feedOptions.SupportedSerializationFormats?.ToString() ?? DefaultSupportedSerializationFormats; return requestHeaders; } @@ -688,23 +684,7 @@ private DocumentFeedResponse GetFeedResponse( { content = memoryStream.ToArray(); } - - IJsonNavigator jsonNavigator = null; - - // Use the users custom navigator first. If it returns null back try the - // internal navigator. - if (this.feedOptions.CosmosSerializationFormatOptions != null) - { - jsonNavigator = this.feedOptions.CosmosSerializationFormatOptions.CreateCustomNavigatorCallback(content); - if (jsonNavigator == null) - { - throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); - } - } - else - { - jsonNavigator = JsonNavigator.Create(content); - } + IJsonNavigator jsonNavigator = JsonNavigator.Create(content); string resourceName = this.GetRootNodeName(documentServiceRequest.ResourceType); diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs index 1c5a9ef21f..cce1a5268a 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs @@ -155,7 +155,6 @@ public override async Task> ExecuteItemQueryAsync( cancellationToken: cancellationToken); return CosmosQueryClientCore.GetCosmosElementResponse( - requestOptions, resourceType, message, trace); @@ -297,7 +296,6 @@ public override void ClearSessionTokenCache(string collectionFullName) } private static TryCatch GetCosmosElementResponse( - QueryRequestOptions requestOptions, ResourceType resourceType, ResponseMessage cosmosResponseMessage, ITrace trace) @@ -336,8 +334,7 @@ private static TryCatch GetCosmosElementResponse( long responseLengthBytes = memoryStream.Length; CosmosArray documents = CosmosQueryClientCore.ParseElementsFromRestStream( memoryStream, - resourceType, - requestOptions.CosmosSerializationFormatOptions); + resourceType); QueryState queryState; if (cosmosResponseMessage.Headers.ContinuationToken != null) @@ -444,12 +441,10 @@ private Task GetRoutingMapProviderAsync() /// /// The memory stream response for the query REST response Azure Cosmos /// The resource type - /// The custom serialization options. This allows custom serialization types like BSON, JSON, or other formats /// An array of CosmosElements parsed from the response body. public static CosmosArray ParseElementsFromRestStream( Stream stream, - ResourceType resourceType, - CosmosSerializationFormatOptions cosmosSerializationOptions) + ResourceType resourceType) { if (!(stream is MemoryStream memoryStream)) { @@ -478,20 +473,7 @@ public static CosmosArray ParseElementsFromRestStream( // You want to create a CosmosElement for each document in "Documents". ReadOnlyMemory content = memoryStream.TryGetBuffer(out ArraySegment buffer) ? buffer : (ReadOnlyMemory)memoryStream.ToArray(); - IJsonNavigator jsonNavigator; - if (cosmosSerializationOptions != null) - { - // Use the users custom navigator - jsonNavigator = cosmosSerializationOptions.CreateCustomNavigatorCallback(content); - if (jsonNavigator == null) - { - throw new InvalidOperationException("The CosmosSerializationOptions did not return a JSON navigator."); - } - } - else - { - jsonNavigator = JsonNavigator.Create(content); - } + IJsonNavigator jsonNavigator = JsonNavigator.Create(content); string resourceName = resourceType switch { diff --git a/Microsoft.Azure.Cosmos/src/ReadFeed/Pagination/ReadFeedPaginationOptions.cs b/Microsoft.Azure.Cosmos/src/ReadFeed/Pagination/ReadFeedPaginationOptions.cs index 757473c02c..ad219dfd99 100644 --- a/Microsoft.Azure.Cosmos/src/ReadFeed/Pagination/ReadFeedPaginationOptions.cs +++ b/Microsoft.Azure.Cosmos/src/ReadFeed/Pagination/ReadFeedPaginationOptions.cs @@ -27,9 +27,8 @@ internal sealed class ReadFeedPaginationOptions : PaginationOptions public ReadFeedPaginationOptions( PaginationDirection? paginationDirection = null, int? pageSizeHint = null, - JsonSerializationFormat? jsonSerializationFormat = null, Dictionary additionalHeaders = null) - : base(pageSizeHint, jsonSerializationFormat, additionalHeaders) + : base(pageSizeHint, additionalHeaders) { this.Direction = paginationDirection; } diff --git a/Microsoft.Azure.Cosmos/src/ReadFeed/ReadFeedCrossFeedRangeAsyncEnumerator.cs b/Microsoft.Azure.Cosmos/src/ReadFeed/ReadFeedCrossFeedRangeAsyncEnumerator.cs index 2f611840f7..95fb8b23cf 100644 --- a/Microsoft.Azure.Cosmos/src/ReadFeed/ReadFeedCrossFeedRangeAsyncEnumerator.cs +++ b/Microsoft.Azure.Cosmos/src/ReadFeed/ReadFeedCrossFeedRangeAsyncEnumerator.cs @@ -45,8 +45,7 @@ public async ValueTask MoveNextAsync() CosmosArray documents = CosmosQueryClientCore.ParseElementsFromRestStream( innerReadFeedPage.Page.Content, - Documents.ResourceType.Document, - cosmosSerializationOptions: null); + Documents.ResourceType.Document); ReadFeedPage page = new ReadFeedPage( documents, innerReadFeedPage.Page.RequestCharge, diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs index 60cea5b48c..f723107ede 100644 --- a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs @@ -5,8 +5,10 @@ namespace Microsoft.Azure.Cosmos { using System; + using System.ComponentModel; using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query; using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Pipeline; using Microsoft.Azure.Documents; @@ -188,6 +190,8 @@ public ConsistencyLevel? ConsistencyLevel internal CosmosSerializationFormatOptions CosmosSerializationFormatOptions { get; set; } + internal SupportedSerializationFormats? SupportedSerializationFormats { get; set; } + internal ExecutionEnvironment? ExecutionEnvironment { get; set; } internal bool? ReturnResultsInDeterministicOrder { get; set; } @@ -240,11 +244,8 @@ internal override void PopulateRequestOptions(RequestMessage request) { request.Headers.Add(HttpConstants.HttpHeaders.ResponseContinuationTokenLimitInKB, this.ResponseContinuationTokenLimitInKb.ToString()); } - - if (this.CosmosSerializationFormatOptions != null) - { - request.Headers.CosmosMessageHeaders.ContentSerializationFormat = this.CosmosSerializationFormatOptions.ContentSerializationFormat; - } + + request.Headers.CosmosMessageHeaders.SupportedSerializationFormats = this.SupportedSerializationFormats?.ToString() ?? DocumentQueryExecutionContextBase.DefaultSupportedSerializationFormats; if (this.StartId != null) { diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationFormatOptions.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationFormatOptions.cs index 81ab5fd1e4..8ebefa502a 100644 --- a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationFormatOptions.cs +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationFormatOptions.cs @@ -52,4 +52,4 @@ public CosmosSerializationFormatOptions( this.CreateCustomWriterCallback = createCustomWriter; } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs index b24ce5f03b..f9ca3f8ecc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs @@ -13,6 +13,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using System.Net; using System.Text; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Cosmos.Tracing; @@ -164,17 +165,17 @@ private void ValidatePageSize(DocumentClient client) public async Task ValidateConsistencyLevelGateway() { DocumentClient client = TestCommon.CreateClient(true); - await ValidateCosistencyLevel(client); + await ValidateConsistencyLevel(client); } [TestMethod] public async Task ValidateConsistencyLevelRntbd() { DocumentClient client = TestCommon.CreateClient(false, Protocol.Tcp); - await ValidateCosistencyLevel(client); + await ValidateConsistencyLevel(client); } - private async Task ValidateCosistencyLevel(DocumentClient client) + private async Task ValidateConsistencyLevel(DocumentClient client) { DocumentCollection collection = TestCommon.CreateOrGetDocumentCollection(client); @@ -203,17 +204,16 @@ private async Task ValidateCosistencyLevel(DocumentClient client) } [TestMethod] - [Owner("brchon")] public void ValidateJsonSerializationFormatGateway() { - var client = TestCommon.CreateClient(true); + using var client = TestCommon.CreateClient(true); ValidateJsonSerializationFormat(client); } [TestMethod] public void ValidateJsonSerializationFormatRntbd() { - var client = TestCommon.CreateClient(false, Protocol.Tcp); + using var client = TestCommon.CreateClient(false, Protocol.Tcp); ValidateJsonSerializationFormat(client); } @@ -306,6 +306,184 @@ private void ValidateJsonSerializationFormatQuery(DocumentClient client, Documen Assert.IsTrue(response.ResponseBody.ReadByte() == HeadersValidationTests.BinarySerializationByteMarkValue); } + [TestMethod] + public void ValidateSupportedSerializationFormatsGateway() + { + using var client = TestCommon.CreateClient(true); + this.ValidateSupportedSerializationFormats(client, true); + } + + [TestMethod] + public void ValidateSupportedSerializationFormatsRntbd() + { + using var client = TestCommon.CreateClient(false, Protocol.Tcp); + this.ValidateSupportedSerializationFormats(client, false); + } + + private void ValidateSupportedSerializationFormats(DocumentClient client, bool isHttps) + { + DocumentCollection collection = TestCommon.CreateOrGetDocumentCollection(client); + this.ValidateSupportedSerializationFormatsReadFeed(client, collection, isHttps); + + List sqlQueryList = new List() + { + new SqlQuerySpec("SELECT * FROM c"), + new SqlQuerySpec("SELECT c.id FROM c ORDER BY c.partitionKey"), + new SqlQuerySpec("SELECT c.name FROM c GROUP BY c.name") + }; + + foreach(SqlQuerySpec sqlQuery in sqlQueryList) + { + this.ValidateSupportedSerializationFormatsQuery(client, collection, sqlQuery, isHttps); + } + } + + private void SupportedSerializationFormatsNegativeCases( + DocumentClient client, + DocumentCollection collection, + string invalidValue, + bool isHttps, + SqlQuerySpec sqlQuerySpec = null) + { + INameValueCollection headers = new RequestNameValueCollection(); + headers.Add(HttpConstants.HttpHeaders.SupportedSerializationFormats, invalidValue); + + try + { + DocumentServiceResponse response; + if (sqlQuerySpec != null) + { + response = this.QueryRequest(client, collection.ResourceId, sqlQuerySpec, headers); + } + else + { + headers.Set(HttpConstants.HttpHeaders.PartitionKey, "[\"test\"]"); + response = this.ReadDocumentFeedRequestSinglePartition(client, collection.ResourceId, headers); + } + + if (isHttps) + { + // Invalid value is treated as default JsonText if HTTPS + Assert.IsTrue(response.StatusCode == HttpStatusCode.OK, "Invalid status code"); + Assert.IsTrue(this.CheckSerializationFormat(response) == JsonSerializationFormat.Text); + } + else + { + Assert.Fail("Should throw an exception"); + } + } + catch (Exception ex) + { + DocumentClientException innerException = ex.InnerException as DocumentClientException; + Assert.IsTrue(innerException.StatusCode == HttpStatusCode.BadRequest, "invalid status code"); + } + } + + private void SupportedSerializationFormatsPositiveCases( + DocumentClient client, + DocumentCollection collection, + SupportedSerializationFormats expectedFormat, + string supportedSerializationFormats, + SqlQuerySpec sqlQuerySpec = null) + { + INameValueCollection headers = new RequestNameValueCollection(); + + headers.Add(HttpConstants.HttpHeaders.SupportedSerializationFormats, supportedSerializationFormats); + DocumentServiceResponse response; + if(sqlQuerySpec!=null) + { + response = this.QueryRequest(client, collection.ResourceId, sqlQuerySpec, headers); + } + else + { + Assert.IsTrue(expectedFormat == SupportedSerializationFormats.JsonText, "ReadFeed response should be in Text"); + response = this.ReadDocumentFeedRequestAsync(client, collection.ResourceId, headers).Result; + } + + Assert.IsTrue(response.StatusCode == HttpStatusCode.OK, "Invalid status code"); + + if(expectedFormat == SupportedSerializationFormats.JsonText) + { + Assert.IsTrue(this.CheckSerializationFormat(response) == JsonSerializationFormat.Text); + } + else + { + Assert.IsTrue(this.CheckSerializationFormat(response) == JsonSerializationFormat.Binary); + } + } + + private void ValidateSupportedSerializationFormatsReadFeed(DocumentClient client, DocumentCollection collection, bool isHttps) + { + // Value not supported + this.SupportedSerializationFormatsNegativeCases(client, collection, "Invalid value", isHttps); + + // Supported values + this.SupportedSerializationFormatsPositiveCases(client, collection, expectedFormat: SupportedSerializationFormats.JsonText, supportedSerializationFormats: "JSONText"); + this.SupportedSerializationFormatsPositiveCases(client, collection, expectedFormat: SupportedSerializationFormats.JsonText, supportedSerializationFormats: "COSMOSbinary"); + this.SupportedSerializationFormatsPositiveCases(client, collection, expectedFormat: SupportedSerializationFormats.JsonText, supportedSerializationFormats: "JsonText, CosmosBinary"); + } + + private void ValidateSupportedSerializationFormatsQuery(DocumentClient client, DocumentCollection collection, SqlQuerySpec sqlQuerySpec, bool isHttps) + { + // Values not supported + this.SupportedSerializationFormatsNegativeCases(client, collection, "Invalid value", isHttps, sqlQuerySpec: sqlQuerySpec); + this.SupportedSerializationFormatsNegativeCases(client, collection, ", ,", isHttps, sqlQuerySpec: sqlQuerySpec); + this.SupportedSerializationFormatsNegativeCases(client, collection, ",,", isHttps, sqlQuerySpec: sqlQuerySpec); + this.SupportedSerializationFormatsNegativeCases(client, collection, "JsonText CosmosBinary", isHttps, sqlQuerySpec: sqlQuerySpec); + this.SupportedSerializationFormatsNegativeCases(client, collection, ",JsonText|CosmosBinary", isHttps, sqlQuerySpec: sqlQuerySpec); + this.SupportedSerializationFormatsNegativeCases(client, collection, "Json Text", isHttps, sqlQuerySpec: sqlQuerySpec); + this.SupportedSerializationFormatsNegativeCases(client, collection, "Json,Text", isHttps, sqlQuerySpec: sqlQuerySpec); + this.SupportedSerializationFormatsNegativeCases(client, collection, "JsonText, ", isHttps, sqlQuerySpec: sqlQuerySpec); + + // Supported values + this.SupportedSerializationFormatsPositiveCases(client, collection, + expectedFormat: SupportedSerializationFormats.JsonText, + supportedSerializationFormats: "jsontext", + sqlQuerySpec: sqlQuerySpec); + /* + this.SupportedSerializationFormatsPositiveCases(client, collection, + expectedFormat: SupportedSerializationFormats.CosmosBinary, + supportedSerializationFormats: "COSMOSBINARY", + sqlQuerySpec: sqlQuerySpec); + */ + this.SupportedSerializationFormatsPositiveCases(client, collection, + expectedFormat: SupportedSerializationFormats.JsonText, + supportedSerializationFormats: "JsonText, CosmosBinary", + sqlQuerySpec: sqlQuerySpec); + /* + this.SupportedSerializationFormatsPositiveCases(client, collection, + expectedFormat: SupportedSerializationFormats.CosmosBinary, + supportedSerializationFormats: "CosmosBinary, HybridRow", + sqlQuerySpec: sqlQuerySpec); + */ + this.SupportedSerializationFormatsPositiveCases(client, collection, + expectedFormat: SupportedSerializationFormats.JsonText, + supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow", + sqlQuerySpec: sqlQuerySpec); + /* + this.SupportedSerializationFormatsPositiveCases(client, collection, + expectedFormat: SupportedSerializationFormats.CosmosBinary, + supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow", + sqlQuerySpec: sqlQuerySpec); + */ + this.SupportedSerializationFormatsPositiveCases(client, collection, + expectedFormat: SupportedSerializationFormats.JsonText, + supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow", + sqlQuerySpec: sqlQuerySpec); + } + + private JsonSerializationFormat CheckSerializationFormat(DocumentServiceResponse response) + { + if(response.ResponseBody.ReadByte() < BinarySerializationByteMarkValue) + { + return JsonSerializationFormat.Text; + } + else + { + return JsonSerializationFormat.Binary; + } + } + [TestMethod] public void ValidateIndexingDirectiveGateway() { @@ -1105,4 +1283,4 @@ internal override string BaseUserAgent } } } -} \ No newline at end of file +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs new file mode 100644 index 0000000000..f4cdab6e51 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs @@ -0,0 +1,102 @@ +namespace Microsoft.Azure.Cosmos.Query +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.EmulatorTests.Query; + using Microsoft.Azure.Cosmos.SDK.EmulatorTests.QueryOracle; + using Microsoft.Azure.Documents; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [TestCategory("Query")] + public sealed class SupportedSerializationFormatsTest : QueryTestsBase + { + [TestMethod] + public async Task TestSupportedSerializationFormats() + { + string[] inputDocuments = new[] + { + @"{""id"":""0"",""name"":""document_0""}", + @"{""id"":""1"",""name"":""document_1""}", + @"{""id"":""2"",""name"":""document_2""}", + @"{""id"":""3"",""name"":""document_3""}", + @"{""id"":""4"",""name"":""document_4""}", + @"{""id"":""5"",""name"":""document_5""}", + @"{""id"":""6"",""name"":""document_6""}", + @"{""id"":""7"",""name"":""document_7""}", + @"{""id"":""8"",""name"":""document_8""}", + @"{""id"":""9"",""name"":""document_9""}", + }; + + await this.CreateIngestQueryDeleteAsync( + ConnectionModes.Direct, + CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, + inputDocuments, + this.TestSupportedSerializationFormatsHelper, + "/id"); + } + + private async Task TestSupportedSerializationFormatsHelper(Container container, IReadOnlyList documents) + { + string[] expectedResults = new[] { "document_0", "document_1", "document_2", "document_3", "document_4", "document_5", "document_6", "document_7", "document_8", "document_9" }; + string query = string.Format("SELECT c.name FROM c"); + List queryRequestOptionsList = new List() + { + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.HybridRow + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText + } + }; + + foreach (QueryRequestOptions requestOptions in queryRequestOptionsList) + { + List results = new List(); + using (FeedIterator feedIterator = container.GetItemQueryIterator(new QueryDefinition(query), requestOptions: requestOptions)) + { + while (feedIterator.HasMoreResults) + { + FeedResponse response = await feedIterator.ReadNextAsync(); + results.AddRange(response.ToList()); + } + } + + string[] actualResults = results + .Select(doc => ((CosmosString)(doc as CosmosObject)["name"]).Value.ToString()) + .ToArray(); + + CollectionAssert.AreEquivalent(expectedResults, actualResults); + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs index c18ac5e420..759e63b345 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs @@ -83,10 +83,6 @@ private async Task RunQueryAsync(Container container) { MaxConcurrency = this.MaxConcurrency, MaxItemCount = this.MaxItemCount, - CosmosSerializationFormatOptions = new CosmosSerializationFormatOptions( - contentSerializationFormat: this.contentSerialization, - createCustomNavigator: (content) => JsonNavigator.Create(content), - createCustomWriter: () => JsonWriter.Create(this.contentSerialization == "JsonText" ? JsonSerializationFormat.Text : JsonSerializationFormat.Binary)) }; if (this.useStronglyTypedIterator) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BinaryEncodingOverTheWireTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BinaryEncodingOverTheWireTests.cs index b55e837844..7d570d65d0 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BinaryEncodingOverTheWireTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BinaryEncodingOverTheWireTests.cs @@ -28,15 +28,11 @@ public class BinaryEncodingOverTheWireTests ConfigurationManager.AppSettings["GatewayEndpoint"], ConfigurationManager.AppSettings["MasterKey"], new CosmosClientOptions() { ConnectionMode = ConnectionMode.Gateway }); - private static readonly CosmosClient DirectHttpsClient = new CosmosClient( - ConfigurationManager.AppSettings["GatewayEndpoint"], - ConfigurationManager.AppSettings["MasterKey"], - new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Documents.Client.Protocol.Https }); private static readonly CosmosClient RntbdClient = new CosmosClient( ConfigurationManager.AppSettings["GatewayEndpoint"], ConfigurationManager.AppSettings["MasterKey"], new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Documents.Client.Protocol.Tcp }); - private static readonly CosmosClient[] Clients = new CosmosClient[] { GatewayClient, DirectHttpsClient, RntbdClient }; + private static readonly CosmosClient[] Clients = new CosmosClient[] { GatewayClient, RntbdClient }; private static readonly CosmosClient Client = RntbdClient; private static readonly AsyncLazy Database = new AsyncLazy(async () => { @@ -124,7 +120,7 @@ public void CheckThatAllTestsAreRunning() [TestMethod] public async Task CombinedScriptsDataTest() { - await this.TestCurratedDocs("CombinedScriptsData.json"); + await this.TestCuratedDocs("CombinedScriptsData.json"); } // For now we are skipping this test since the documents are too large to ingest and we get a rate size too large (HTTP 413). @@ -139,96 +135,96 @@ public async Task CountriesTest() [TestMethod] public async Task DevTestCollTest() { - await this.TestCurratedDocs("devtestcoll.json"); + await this.TestCuratedDocs("devtestcoll.json"); } [TestMethod] public async Task LastFMTest() { - await this.TestCurratedDocs("lastfm"); + await this.TestCuratedDocs("lastfm"); } [TestMethod] public async Task LogDataTest() { - await this.TestCurratedDocs("LogData.json"); + await this.TestCuratedDocs("LogData.json"); } [TestMethod] public async Task MillionSong1KDocumentsTest() { - await this.TestCurratedDocs("MillionSong1KDocuments.json"); + await this.TestCuratedDocs("MillionSong1KDocuments.json"); } [TestMethod] public async Task MsnCollectionTest() { - await this.TestCurratedDocs("MsnCollection.json"); + await this.TestCuratedDocs("MsnCollection.json"); } [TestMethod] public async Task NutritionDataTest() { - await this.TestCurratedDocs("NutritionData"); + await this.TestCuratedDocs("NutritionData"); } [TestMethod] public async Task RunsCollectionTest() { - await this.TestCurratedDocs("runsCollection"); + await this.TestCuratedDocs("runsCollection"); } [TestMethod] public async Task StatesCommitteesTest() { - await this.TestCurratedDocs("states_committees.json"); + await this.TestCuratedDocs("states_committees.json"); } [TestMethod] public async Task StatesLegislatorsTest() { - await this.TestCurratedDocs("states_legislators"); + await this.TestCuratedDocs("states_legislators"); } [TestMethod] public async Task Store01Test() { - await this.TestCurratedDocs("store01C.json"); + await this.TestCuratedDocs("store01C.json"); } [TestMethod] public async Task TicinoErrorBucketsTest() { - await this.TestCurratedDocs("TicinoErrorBuckets"); + await this.TestCuratedDocs("TicinoErrorBuckets"); } [TestMethod] public async Task TwitterDataTest() { - await this.TestCurratedDocs("twitter_data"); + await this.TestCuratedDocs("twitter_data"); } [TestMethod] public async Task Ups1Test() { - await this.TestCurratedDocs("ups1"); + await this.TestCuratedDocs("ups1"); } [TestMethod] public async Task XpertEventsTest() { - await this.TestCurratedDocs("XpertEvents"); + await this.TestCuratedDocs("XpertEvents"); } - private async Task TestCurratedDocs(string path) + private async Task TestCuratedDocs(string path) { IEnumerable documents = BinaryEncodingOverTheWireTests.GetDocumentsFromCurratedDoc(path); await BinaryEncodingOverTheWireTests.CreateIngestQueryDelete( documents.Select(x => x.ToString()), - this.TestCurratedDocs); + this.TestCuratedDocs); } - private async Task TestCurratedDocs(CosmosClient cosmosClient, Container container, List items) + private async Task TestCuratedDocs(CosmosClient cosmosClient, Container container, List items) { HashSet inputItems = new HashSet(items, JsonTokenEqualityComparer.Value); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ContinuationTokens/OrderByQueryResultTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ContinuationTokens/OrderByQueryResultTests.cs index 86093aa296..7207f897be 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ContinuationTokens/OrderByQueryResultTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ContinuationTokens/OrderByQueryResultTests.cs @@ -31,11 +31,7 @@ public void TestOrderByUndefined() CosmosArray documents = CosmosQueryClientCore.ParseElementsFromRestStream( memoryStream, - Documents.ResourceType.Document, - new CosmosSerializationFormatOptions( - "JsonText", - (content) => JsonNavigator.Create(content), - () => JsonWriter.Create(JsonSerializationFormat.Text))); + Documents.ResourceType.Document); List orderByQueryResults = documents.Select(x => new OrderByQueryResult(x)).ToList(); Assert.AreEqual(14, orderByQueryResults.Count); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/FeedOptionTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/FeedOptionTests.cs index 54c965cce8..703636ca81 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/FeedOptionTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/FeedOptionTests.cs @@ -50,6 +50,31 @@ public void TestCopyConstructor() _ = new FeedOptions(fo); } + [TestMethod] + public async Task TestSupportedSerializationFormats() + { + FeedOptions feedOptions = new FeedOptions(); + Mock client = new Mock(MockBehavior.Strict); + client.Setup(x => x.GetDefaultConsistencyLevelAsync()).Returns(Task.FromResult(ConsistencyLevel.BoundedStaleness)); + client.Setup(x => x.GetDesiredConsistencyLevelAsync()).Returns(Task.FromResult(ConsistencyLevel.BoundedStaleness)); + Expression> randomFunc = x => x * 2; + + TestQueryExecutionContext testQueryExecutionContext = new TestQueryExecutionContext( + client.Object, + ResourceType.Document, + typeof(TestQueryExecutionContext), + randomFunc, + feedOptions, + string.Empty, + true, Guid.NewGuid()); + INameValueCollection headers = await testQueryExecutionContext.CreateCommonHeadersAsync(feedOptions); + Assert.AreEqual("JsonText,CosmosBinary", headers[HttpConstants.HttpHeaders.SupportedSerializationFormats]); + + feedOptions.SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow; + headers = await testQueryExecutionContext.CreateCommonHeadersAsync(feedOptions); + Assert.AreEqual("CosmosBinary, HybridRow", headers[HttpConstants.HttpHeaders.SupportedSerializationFormats]); + } + internal class TestQueryExecutionContext : DocumentQueryExecutionContextBase { public TestQueryExecutionContext(