Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Internal] Query : Fixes response parsing error for ReadFeed by limiting binary format to query operations #4515

Merged
merged 8 commits into from
May 29, 2024
Merged

[Internal] Query : Fixes response parsing error for ReadFeed by limiting binary format to query operations #4515

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
887142f
Limit binary format to query operations
adityasa May 24, 2024
77127f0
Addressed comments, updated test.
adityasa May 25, 2024
ac6bce9
Updated coverage to include query stream API.
adityasa May 25, 2024
95e944e
Test Change.
adityasa May 28, 2024
f0b93e9
Test change 2
adityasa May 28, 2024
da7ae54
ReadFeed Encryption tests only with project reference
ealsur May 28, 2024
7c29b01
Revert
ealsur May 28, 2024
39d6a01
Removed comment
ealsur May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,14 @@ internal override void PopulateRequestOptions(RequestMessage request)
{
request.Headers.Add(HttpConstants.HttpHeaders.ResponseContinuationTokenLimitInKB, this.ResponseContinuationTokenLimitInKb.ToString());
}

request.Headers.CosmosMessageHeaders.SupportedSerializationFormats = this.SupportedSerializationFormats?.ToString() ?? DocumentQueryExecutionContextBase.DefaultSupportedSerializationFormats;

// All query APIs (GetItemQueryIterator, GetItemLinqQueryable and GetItemQueryStreamIterator) turn into ReadFeed operation if query text is null.
// In such a case, query pipelines are still involved (including QueryRequestOptions). In general backend only honors SupportedSerializationFormats
// for OperationType Query but has a bug where it returns a binary response for ReadFeed API when partition key is also specified in the request.
if (request.OperationType == OperationType.Query)
adityasa marked this conversation as resolved.
Show resolved Hide resolved
{
request.Headers.CosmosMessageHeaders.SupportedSerializationFormats = this.SupportedSerializationFormats?.ToString() ?? DocumentQueryExecutionContextBase.DefaultSupportedSerializationFormats;
}

if (this.StartId != null)
{
Expand Down
14 changes: 4 additions & 10 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/HeadersValidationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ private void SupportedSerializationFormatsPositiveCases(
INameValueCollection headers = new RequestNameValueCollection();

headers.Add(HttpConstants.HttpHeaders.SupportedSerializationFormats, supportedSerializationFormats);
DocumentServiceResponse response;
DocumentServiceResponse response;
if(sqlQuerySpec!=null)
{
response = this.QueryRequest(client, collection.ResourceId, sqlQuerySpec, headers);
Expand Down Expand Up @@ -440,34 +440,28 @@ private void ValidateSupportedSerializationFormatsQuery(DocumentClient client, D
expectedFormat: SupportedSerializationFormats.JsonText,
supportedSerializationFormats: "jsontext",
sqlQuerySpec: sqlQuerySpec);
/*
this.SupportedSerializationFormatsPositiveCases(client, collection,
expectedFormat: SupportedSerializationFormats.CosmosBinary,
supportedSerializationFormats: "COSMOSBINARY",
sqlQuerySpec: sqlQuerySpec);
*/
this.SupportedSerializationFormatsPositiveCases(client, collection,
expectedFormat: SupportedSerializationFormats.JsonText,
expectedFormat: SupportedSerializationFormats.CosmosBinary,
supportedSerializationFormats: "JsonText, CosmosBinary",
sqlQuerySpec: sqlQuerySpec);
/*
this.SupportedSerializationFormatsPositiveCases(client, collection,
expectedFormat: SupportedSerializationFormats.CosmosBinary,
supportedSerializationFormats: "CosmosBinary, HybridRow",
sqlQuerySpec: sqlQuerySpec);
*/
this.SupportedSerializationFormatsPositiveCases(client, collection,
expectedFormat: SupportedSerializationFormats.JsonText,
expectedFormat: SupportedSerializationFormats.CosmosBinary,
supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow",
sqlQuerySpec: sqlQuerySpec);
/*
this.SupportedSerializationFormatsPositiveCases(client, collection,
expectedFormat: SupportedSerializationFormats.CosmosBinary,
supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow",
sqlQuerySpec: sqlQuerySpec);
*/
this.SupportedSerializationFormatsPositiveCases(client, collection,
expectedFormat: SupportedSerializationFormats.JsonText,
expectedFormat: SupportedSerializationFormats.CosmosBinary,
supportedSerializationFormats: "JsonText, CosmosBinary, HybridRow",
sqlQuerySpec: sqlQuerySpec);
}
Expand Down
157 changes: 108 additions & 49 deletions ...mos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SupportedSerializationFormatsTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
namespace Microsoft.Azure.Cosmos.Query
namespace Microsoft.Azure.Cosmos.EmulatorTests.Query
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.CosmosElements;
using Microsoft.Azure.Cosmos.EmulatorTests.Query;
using Microsoft.Azure.Cosmos.SDK.EmulatorTests.QueryOracle;
using Microsoft.Azure.Documents;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json.Linq;

[TestClass]
[TestCategory("Query")]
Expand Down Expand Up @@ -36,66 +35,126 @@ await this.CreateIngestQueryDeleteAsync(
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocuments,
this.TestSupportedSerializationFormatsHelper,
"/id");
"/id");
}

private async Task TestSupportedSerializationFormatsHelper(Container container, IReadOnlyList<CosmosObject> documents)
{
string[] expectedResults = new[] { "document_0", "document_1", "document_2", "document_3", "document_4", "document_5", "document_6", "document_7", "document_8", "document_9" };
string query = string.Format("SELECT c.name FROM c");
List<QueryRequestOptions> queryRequestOptionsList = new List<QueryRequestOptions>()
{
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.HybridRow
},
new QueryRequestOptions()
List<(Cosmos.PartitionKey?, string[]) > partitionKeyAndExpectedResults = new List<(Cosmos.PartitionKey? partitionKey, string[] documents)>
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText
},
new QueryRequestOptions()
(
partitionKey: null,
documents: new[] { "document_0", "document_1", "document_2", "document_3", "document_4","document_5", "document_6", "document_7", "document_8", "document_9" }
),
(
partitionKey: new Cosmos.PartitionKey("0"),
documents: new[] { "document_0" }
)
};

foreach (string query in new string[]
{
SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary
},
new QueryRequestOptions()
null, // With null query, this turns into a ReadFeed API, for which SupportedSerializationFormats should be ignored.
"SELECT * FROM c"
})
{
List<QueryRequestOptions> queryRequestOptionsList = new List<QueryRequestOptions>()
{
SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary
},
new QueryRequestOptions()
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.HybridRow
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText | SupportedSerializationFormats.CosmosBinary | SupportedSerializationFormats.HybridRow
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.CosmosBinary
},
new QueryRequestOptions()
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText
}
};

// GetItemQueryIterator
foreach (QueryRequestOptions requestOptions in queryRequestOptionsList)
{
SupportedSerializationFormats = SupportedSerializationFormats.JsonText
QueryDefinition queryDefinition = query != null ? new QueryDefinition(query) : null;
foreach ((Cosmos.PartitionKey? partitionKey, string[] expectedResults) in partitionKeyAndExpectedResults)
{
requestOptions.PartitionKey = partitionKey;

List<JObject> queryResults = new List<JObject>();
using (FeedIterator<JObject> feedIterator = container.GetItemQueryIterator<JObject>(queryDefinition, requestOptions: requestOptions))
{
while (feedIterator.HasMoreResults)
{
FeedResponse<JObject> response = await feedIterator.ReadNextAsync();
queryResults.AddRange(response.ToList());
}
}

string[] actualResults = queryResults
.Select(doc => doc["name"].ToString())
.ToArray();

CollectionAssert.AreEquivalent(expectedResults, actualResults);
}
}
};

foreach (QueryRequestOptions requestOptions in queryRequestOptionsList)
{
List<CosmosElement> results = new List<CosmosElement>();
using (FeedIterator<CosmosElement> feedIterator = container.GetItemQueryIterator<CosmosElement>(new QueryDefinition(query), requestOptions: requestOptions))
// GetItemQueryStreamIterator
foreach (QueryRequestOptions requestOptions in queryRequestOptionsList)
{
while (feedIterator.HasMoreResults)
QueryDefinition queryDefinition = query != null ? new QueryDefinition(query) : null;
foreach ((Cosmos.PartitionKey? partitionKey, string[] expectedResults) in partitionKeyAndExpectedResults)
{
FeedResponse<CosmosElement> response = await feedIterator.ReadNextAsync();
results.AddRange(response.ToList());
requestOptions.PartitionKey = partitionKey;

List<CosmosElement> queryResults = new List<CosmosElement>();
using (FeedIterator feedIterator = container.GetItemQueryStreamIterator(queryDefinition, requestOptions: requestOptions))
{
while (feedIterator.HasMoreResults)
{
ResponseMessage response = await feedIterator.ReadNextAsync();
queryResults.AddRange(Deserialize(response.Content));
}
}

string[] actualResults = queryResults
.Select(doc => ((CosmosString)((CosmosObject)doc)["name"]).Value.ToString())
.ToArray();

CollectionAssert.AreEquivalent(expectedResults, actualResults);
}
}
}
}

string[] actualResults = results
.Select(doc => ((CosmosString)(doc as CosmosObject)["name"]).Value.ToString())
.ToArray();

CollectionAssert.AreEquivalent(expectedResults, actualResults);
private static IEnumerable<CosmosElement> Deserialize(Stream content)
{
string contentAsString = new StreamReader(content).ReadToEnd();
CosmosObject obj = CosmosObject.Parse(contentAsString);
foreach (CosmosElement element in (CosmosArray)obj["Documents"])
{
yield return element;
}
}
}
Expand Down
Loading