From 887142fa762c8d988d51a39bb8986bd900a5a858 Mon Sep 17 00:00:00 2001 From: "REDMOND\\adityasa" Date: Fri, 24 May 2024 15:55:42 -0700 Subject: [PATCH 1/8] Limit binary format to query operations --- .../src/RequestOptions/QueryRequestOptions.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs index 88db939c96..11669f6b72 100644 --- a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs @@ -237,8 +237,11 @@ internal override void PopulateRequestOptions(RequestMessage request) { request.Headers.Add(HttpConstants.HttpHeaders.ResponseContinuationTokenLimitInKB, this.ResponseContinuationTokenLimitInKb.ToString()); } - - request.Headers.CosmosMessageHeaders.SupportedSerializationFormats = this.SupportedSerializationFormats?.ToString() ?? DocumentQueryExecutionContextBase.DefaultSupportedSerializationFormats; + + if (request.OperationType == OperationType.Query) + { + request.Headers.CosmosMessageHeaders.SupportedSerializationFormats = this.SupportedSerializationFormats?.ToString() ?? DocumentQueryExecutionContextBase.DefaultSupportedSerializationFormats; + } if (this.StartId != null) { From 77127f0bc772988ad36f32083ffdee233429b72f Mon Sep 17 00:00:00 2001 From: "REDMOND\\adityasa" Date: Sat, 25 May 2024 13:28:01 -0700 Subject: [PATCH 2/8] Addressed comments, updated test. --- .../src/RequestOptions/QueryRequestOptions.cs | 3 + .../HeadersValidationTests.cs | 14 +- .../SupportedSerializationFormatsTest.cs | 122 +++++++++++------- 3 files changed, 79 insertions(+), 60 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs index 11669f6b72..bbb019f08b 100644 --- a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs @@ -238,6 +238,9 @@ internal override void PopulateRequestOptions(RequestMessage request) request.Headers.Add(HttpConstants.HttpHeaders.ResponseContinuationTokenLimitInKB, this.ResponseContinuationTokenLimitInKb.ToString()); } + // All query APIs (GetItemQueryIterator, GetItemLinqQueryable and GetItemQueryStreamIterator) turn into ReadFeed operation if query text is null. + // In such a case, query pipelines are still involved (including QueryRequestOptions). In general backend only honors SupportedSerializationFormats + // for OperationType Query but has a bug where it returns a binary response for ReadFeed API when partition key is also specified in the request. if (request.OperationType == OperationType.Query) { request.Headers.CosmosMessageHeaders.SupportedSerializationFormats = this.SupportedSerializationFormats?.ToString() ?? DocumentQueryExecutionContextBase.DefaultSupportedSerializationFormats; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs index f9ca3f8ecc..64551c009b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs @@ -389,7 +389,7 @@ private void SupportedSerializationFormatsPositiveCases( INameValueCollection headers = new RequestNameValueCollection(); headers.Add(HttpConstants.HttpHeaders.SupportedSerializationFormats, supportedSerializationFormats); - DocumentServiceResponse response; + DocumentServiceResponse response; if(sqlQuerySpec!=null) { response = this.QueryRequest(client, collection.ResourceId, sqlQuerySpec, headers); @@ -440,34 +440,28 @@ private void ValidateSupportedSerializationFormatsQuery(DocumentClient client, D expectedFormat: SupportedSerializationFormats.JsonText, supportedSerializationFormats: "jsontext", sqlQuerySpec: sqlQuerySpec); - /* this.SupportedSerializationFormatsPositiveCases(client, collection, expectedFormat: SupportedSerializationFormats.CosmosBinary, supportedSerializationFormats: "COSMOSBINARY", sqlQuerySpec: sqlQuerySpec); - */ this.SupportedSerializationFormatsPositiveCases(client, collection, - expectedFormat: SupportedSerializationFormats.JsonText, + expectedFormat: SupportedSerializationFormats.CosmosBinary, supportedSerializationFormats: "JsonText, CosmosBinary", sqlQuerySpec: sqlQuerySpec); - /* this.SupportedSerializationFormatsPositiveCases(client, collection, expectedFormat: SupportedSerializationFormats.CosmosBinary, supportedSerializationFormats: "CosmosBinary, HybridRow", sqlQuerySpec: sqlQuerySpec); - */ this.SupportedSerializationFormatsPositiveCases(client, collection, - expectedFormat: SupportedSerializationFormats.JsonText, + expectedFormat: SupportedSerializationFormats.CosmosBinary, supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow", sqlQuerySpec: sqlQuerySpec); - /* this.SupportedSerializationFormatsPositiveCases(client, collection, expectedFormat: SupportedSerializationFormats.CosmosBinary, supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow", sqlQuerySpec: sqlQuerySpec); - */ this.SupportedSerializationFormatsPositiveCases(client, collection, - expectedFormat: SupportedSerializationFormats.JsonText, + expectedFormat: SupportedSerializationFormats.CosmosBinary, supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow", sqlQuerySpec: sqlQuerySpec); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs index f4cdab6e51..8c27e415e9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs @@ -1,14 +1,13 @@ namespace Microsoft.Azure.Cosmos.Query { - using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.EmulatorTests.Query; - using Microsoft.Azure.Cosmos.SDK.EmulatorTests.QueryOracle; using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json.Linq; [TestClass] [TestCategory("Query")] @@ -36,66 +35,89 @@ await this.CreateIngestQueryDeleteAsync( CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, inputDocuments, this.TestSupportedSerializationFormatsHelper, - "/id"); + "/id"); } private async Task TestSupportedSerializationFormatsHelper(Container container, IReadOnlyList documents) { - string[] expectedResults = new[] { "document_0", "document_1", "document_2", "document_3", "document_4", "document_5", "document_6", "document_7", "document_8", "document_9" }; - string query = string.Format("SELECT c.name FROM c"); - List queryRequestOptionsList = new List() - { - new QueryRequestOptions() - { - SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow - }, - new QueryRequestOptions() - { - SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary - }, - new QueryRequestOptions() - { - SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.HybridRow - }, - new QueryRequestOptions() - { - SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow - }, - new QueryRequestOptions() - { - SupportedSerializationFormats = SupportedSerializationFormats.JsonText - }, - new QueryRequestOptions() - { - SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary - }, - new QueryRequestOptions() + List<(Cosmos.PartitionKey?, string[]) > partitionKeyAndExpectedResults = new List<(Cosmos.PartitionKey? partitionKey, string[] documents)> { - SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary - }, - new QueryRequestOptions() - { - SupportedSerializationFormats = SupportedSerializationFormats.JsonText - } - }; + ( + partitionKey: null, + documents: new[] { "document_0", "document_1", "document_2", "document_3", "document_4", "document_5", "document_6", "document_7", "document_8", "document_9" } + ), + ( + partitionKey: new Cosmos.PartitionKey("0"), + documents: new[] { "document_0" } + ) + }; - foreach (QueryRequestOptions requestOptions in queryRequestOptionsList) + foreach (string query in new string[] + { + null, // With null query, this turns into a ReadFeed API, for which SupportedSerializationFormats should be ignored. + "SELECT * FROM c" + }) { - List results = new List(); - using (FeedIterator feedIterator = container.GetItemQueryIterator(new QueryDefinition(query), requestOptions: requestOptions)) + List queryRequestOptionsList = new List() { - while (feedIterator.HasMoreResults) + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary + }, + new QueryRequestOptions() { - FeedResponse response = await feedIterator.ReadNextAsync(); - results.AddRange(response.ToList()); + SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.HybridRow + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary + }, + new QueryRequestOptions() + { + SupportedSerializationFormats = SupportedSerializationFormats.JsonText } - } + }; - string[] actualResults = results - .Select(doc => ((CosmosString)(doc as CosmosObject)["name"]).Value.ToString()) - .ToArray(); + foreach (QueryRequestOptions requestOptions in queryRequestOptionsList) + { + QueryDefinition queryDefinition = query != null ? new QueryDefinition(query) : null; + foreach ((Cosmos.PartitionKey? partitionKey, string[] expectedResults) in partitionKeyAndExpectedResults) + { + requestOptions.PartitionKey = partitionKey; + + List queryResults = new List(); + using (FeedIterator feedIterator = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions)) + { + while (feedIterator.HasMoreResults) + { + FeedResponse response = await feedIterator.ReadNextAsync(); + queryResults.AddRange(response.ToList()); + } + } - CollectionAssert.AreEquivalent(expectedResults, actualResults); + string[] actualResults = queryResults + .Select(doc => doc["name"].ToString()) + .ToArray(); + + CollectionAssert.AreEquivalent(expectedResults, actualResults); + } + } } } } From ac6bce92300c776de9ba740fdddad63112d885f6 Mon Sep 17 00:00:00 2001 From: "REDMOND\\adityasa" Date: Sat, 25 May 2024 15:12:33 -0700 Subject: [PATCH 3/8] Updated coverage to include query stream API. --- .../SupportedSerializationFormatsTest.cs | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs index 8c27e415e9..1c4dc9854a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs @@ -1,10 +1,10 @@ -namespace Microsoft.Azure.Cosmos.Query +namespace Microsoft.Azure.Cosmos.EmulatorTests.Query { using System.Collections.Generic; + using System.IO; using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.CosmosElements; - using Microsoft.Azure.Cosmos.EmulatorTests.Query; using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json.Linq; @@ -44,7 +44,7 @@ private async Task TestSupportedSerializationFormatsHelper(Container container, { ( partitionKey: null, - documents: new[] { "document_0", "document_1", "document_2", "document_3", "document_4", "document_5", "document_6", "document_7", "document_8", "document_9" } + documents: new[] { "document_0", "document_1", "document_2", "document_3", "document_4","document_5", "document_6", "document_7", "document_8", "document_9" } ), ( partitionKey: new Cosmos.PartitionKey("0"), @@ -94,6 +94,7 @@ private async Task TestSupportedSerializationFormatsHelper(Container container, } }; + // GetItemQueryIterator foreach (QueryRequestOptions requestOptions in queryRequestOptionsList) { QueryDefinition queryDefinition = query != null ? new QueryDefinition(query) : null; @@ -118,6 +119,42 @@ private async Task TestSupportedSerializationFormatsHelper(Container container, CollectionAssert.AreEquivalent(expectedResults, actualResults); } } + + // GetItemQueryStreamIterator + foreach (QueryRequestOptions requestOptions in queryRequestOptionsList) + { + QueryDefinition queryDefinition = query != null ? new QueryDefinition(query) : null; + foreach ((Cosmos.PartitionKey? partitionKey, string[] expectedResults) in partitionKeyAndExpectedResults) + { + requestOptions.PartitionKey = partitionKey; + + List queryResults = new List(); + using (FeedIterator feedIterator = container.GetItemQueryStreamIterator(queryDefinition, requestOptions: requestOptions)) + { + while (feedIterator.HasMoreResults) + { + ResponseMessage response = await feedIterator.ReadNextAsync(); + queryResults.AddRange(Deserialize(response.Content)); + } + } + + string[] actualResults = queryResults + .Select(doc => ((CosmosString)((CosmosObject)doc)["name"]).Value.ToString()) + .ToArray(); + + CollectionAssert.AreEquivalent(expectedResults, actualResults); + } + } + } + } + + private static IEnumerable Deserialize(Stream content) + { + string contentAsString = new StreamReader(content).ReadToEnd(); + CosmosObject obj = CosmosObject.Parse(contentAsString); + foreach (CosmosElement element in (CosmosArray)obj["Documents"]) + { + yield return element; } } } From 95e944e9c829990f4c3b60f5425b58cae0811478 Mon Sep 17 00:00:00 2001 From: "REDMOND\\adityasa" Date: Mon, 27 May 2024 21:42:30 -0700 Subject: [PATCH 4/8] Test Change. --- .../src/Query/v2Query/DocumentQueryExecutionContextBase.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs index 5bd47c8a21..1b3a8defae 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs @@ -32,8 +32,9 @@ namespace Microsoft.Azure.Cosmos.Query internal abstract class DocumentQueryExecutionContextBase : IDocumentQueryExecutionContext { - public static readonly string DefaultSupportedSerializationFormats = string.Join(",", SupportedSerializationFormats.JsonText, SupportedSerializationFormats.CosmosBinary); - + // ISSUE-TODO-adityasa-2024/5/27 - Re-enable binary format. Currently encryption codepath is not able to handle binary response. + public static readonly string DefaultSupportedSerializationFormats = string.Join(",", SupportedSerializationFormats.JsonText); + public readonly struct InitParams { public IDocumentQueryClient Client { get; } From f0b93e963b6086bd49c257e3aafbea317fd14e0f Mon Sep 17 00:00:00 2001 From: "REDMOND\\adityasa" Date: Mon, 27 May 2024 22:05:58 -0700 Subject: [PATCH 5/8] Test change 2 --- .../src/Query/v2Query/DocumentQueryExecutionContextBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs index 1b3a8defae..8e44aa5606 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs @@ -33,7 +33,7 @@ namespace Microsoft.Azure.Cosmos.Query internal abstract class DocumentQueryExecutionContextBase : IDocumentQueryExecutionContext { // ISSUE-TODO-adityasa-2024/5/27 - Re-enable binary format. Currently encryption codepath is not able to handle binary response. - public static readonly string DefaultSupportedSerializationFormats = string.Join(",", SupportedSerializationFormats.JsonText); + public static readonly string DefaultSupportedSerializationFormats = SupportedSerializationFormats.JsonText.ToString(); public readonly struct InitParams { From da7ae54b4747587e0b973266bb9a3094198ef470 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 28 May 2024 10:19:38 -0700 Subject: [PATCH 6/8] ReadFeed Encryption tests only with project reference --- .../tests/EmulatorTests/MdeEncryptionTests.cs | 3 +++ .../Microsoft.Azure.Cosmos.Encryption.EmulatorTests.csproj | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs index a084dcdab4..853a82aba7 100644 --- a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs @@ -581,10 +581,13 @@ public async Task EncryptionCreateItemAndQuery() TestDoc expectedDoc = new TestDoc(testDoc); +#if SDKPROJECTREF + TODO: Re-enable when new SDK is referenced await MdeEncryptionTests.ValidateQueryResultsAsync( MdeEncryptionTests.encryptionContainer, query: null, expectedDocList: new List { expectedDoc }); +#endif expectedDoc = new TestDoc(testDoc); await MdeEncryptionTests.ValidateQueryResultsAsync( diff --git a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.EmulatorTests.csproj b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.EmulatorTests.csproj index 0b7c24dbdd..667ad7b93a 100644 --- a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.EmulatorTests.csproj @@ -68,4 +68,8 @@ $(DefineConstants);ENCRYPTIONTESTPREVIEW + + + $(DefineConstants);SDKPROJECTREF + From 7c29b0158237e477aafdbbc6b6f8bb11b6cb807f Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 28 May 2024 10:40:02 -0700 Subject: [PATCH 7/8] Revert --- .../src/Query/v2Query/DocumentQueryExecutionContextBase.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs index 8e44aa5606..5bd47c8a21 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v2Query/DocumentQueryExecutionContextBase.cs @@ -32,9 +32,8 @@ namespace Microsoft.Azure.Cosmos.Query internal abstract class DocumentQueryExecutionContextBase : IDocumentQueryExecutionContext { - // ISSUE-TODO-adityasa-2024/5/27 - Re-enable binary format. Currently encryption codepath is not able to handle binary response. - public static readonly string DefaultSupportedSerializationFormats = SupportedSerializationFormats.JsonText.ToString(); - + public static readonly string DefaultSupportedSerializationFormats = string.Join(",", SupportedSerializationFormats.JsonText, SupportedSerializationFormats.CosmosBinary); + public readonly struct InitParams { public IDocumentQueryClient Client { get; } From 39d6a01f421b307912bf8c64cfab0f0b16684bab Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 28 May 2024 10:52:02 -0700 Subject: [PATCH 8/8] Removed comment --- .../tests/EmulatorTests/MdeEncryptionTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs index 853a82aba7..bb6d6e4919 100644 --- a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs @@ -582,7 +582,6 @@ public async Task EncryptionCreateItemAndQuery() TestDoc expectedDoc = new TestDoc(testDoc); #if SDKPROJECTREF - TODO: Re-enable when new SDK is referenced await MdeEncryptionTests.ValidateQueryResultsAsync( MdeEncryptionTests.encryptionContainer, query: null,