Query : Adds support for newtonsoft member access via ExtensionData (#…

…3834) * Support newtonsoft member access via ExtensionData * Return null instead of empty string * Added tests for select & where * Updated baseline with note --------- Co-authored-by: leminh98 <[email protected]>
Azure · Jun 26, 2023 · abab80e · abab80e
1 parent 3f4b7bd
commit abab80e
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 1 deletion.
diff --git a/Microsoft.Azure.Cosmos/src/Linq/ExpressionToSQL.cs b/Microsoft.Azure.Cosmos/src/Linq/ExpressionToSQL.cs
@@ -742,6 +742,12 @@ private static SqlScalarExpression VisitMemberAccess(MemberExpression inputExpre
             SqlScalarExpression memberExpression = ExpressionToSql.VisitScalarExpression(inputExpression.Expression, context);
             string memberName = inputExpression.Member.GetMemberName(context.linqSerializerOptions);
 
+            // If the resulting memberName is null, then the indexer should be on the root of the object.
+            if (memberName == null)
+            {
+                return memberExpression;
+            }
+
             // if expression is nullable
             if (inputExpression.Expression.Type.IsNullable())
             {

diff --git a/Microsoft.Azure.Cosmos/src/Linq/TypeSystem.cs b/Microsoft.Azure.Cosmos/src/Linq/TypeSystem.cs
@@ -25,6 +25,14 @@ public static Type GetElementType(Type type)
         public static string GetMemberName(this MemberInfo memberInfo, CosmosLinqSerializerOptions linqSerializerOptions = null)
         {
             string memberName = null;
+
+            // Check if Newtonsoft JsonExtensionDataAttribute is present on the member, if so, return empty member name.
+            JsonExtensionDataAttribute jsonExtensionDataAttribute = memberInfo.GetCustomAttribute<JsonExtensionDataAttribute>(true);
+            if (jsonExtensionDataAttribute != null && jsonExtensionDataAttribute.ReadData)
+            {
+                return null;
+            }
+
             // Json.Net honors JsonPropertyAttribute more than DataMemberAttribute
             // So we check for JsonPropertyAttribute first.
             JsonPropertyAttribute jsonPropertyAttribute = memberInfo.GetCustomAttribute<JsonPropertyAttribute>(true);

diff --git a/...Tests/BaselineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslation.xml b/...Tests/BaselineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslation.xml
@@ -268,6 +268,17 @@ FROM root]]></SqlQuery>
     <Output>
       <SqlQuery><![CDATA[
 SELECT VALUE root["x"] 
+FROM root]]></SqlQuery>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Select extension data]]></Description>
+      <Expression><![CDATA[query.Select(x => new complex() {NewtonsoftExtensionData = new Dictionary`2() {Void Add(System.String, System.Object)("test", Convert(1.5, Object))}, NetExtensionData = new Dictionary`2() {Void Add(System.String, System.Object)("OtherTest", Convert(1.5, Object))}})]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[
+SELECT VALUE {"dbl": 0, "str": null, "b": false, "dblArray": null, "inside": {"x": 0, "y": 0, "id": null, "pk": null}, "id": null, "pk": null, "NetExtensionData": {"OtherTest": 1.5}, "test": 1.5} 
 FROM root]]></SqlQuery>
     </Output>
   </Result>

diff --git a/...ineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslationComplexData.xml b/...ineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslationComplexData.xml
@@ -166,4 +166,38 @@ FROM root
 JOIN x0 IN root["dblArray"]]]></SqlQuery>
     </Output>
   </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Select where extensiondata]]></Description>
+      <Expression><![CDATA[query.Where(p => (Convert(p.NewtonsoftExtensionData.get_Item("age"), Int32) > 18)).Select(x => new AnonymousType(Age = Convert(x.NewtonsoftExtensionData.get_Item("age"), Int32)))]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[
+SELECT VALUE {"Age": root["age"]} 
+FROM root 
+WHERE (root["age"] > 18)]]></SqlQuery>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Select where extensiondata contains]]></Description>
+      <Expression><![CDATA[query.Where(p => Convert(p.NewtonsoftExtensionData.get_Item("tags"), String[]).Contains("item-1")).Select(x => Convert(x.NewtonsoftExtensionData.get_Item("tags"), String[]))]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[
+SELECT VALUE root["tags"] 
+FROM root 
+WHERE ARRAY_CONTAINS(root["tags"], "item-1")]]></SqlQuery>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[SelectMany where extensiondata contains]]></Description>
+      <Expression><![CDATA[query.Where(p => Convert(p.NewtonsoftExtensionData.get_Item("tags"), String[]).Contains("item-1")).SelectMany(x => Convert(x.NewtonsoftExtensionData.get_Item("tags"), Object[]))]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[]]></SqlQuery>
+      <ErrorMessage><![CDATA[Expression with NodeType 'Convert' is not supported.]]></ErrorMessage>
+    </Output>
+  </Result>
 </Results>
diff --git a/...zure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqSQLTranslationBaselineTests.cs b/...zure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqSQLTranslationBaselineTests.cs
@@ -103,6 +103,12 @@ struct complex
             public string id;
             public string pk;
 
+            [Newtonsoft.Json.JsonExtensionData(ReadData = true, WriteData = true)]
+            public Dictionary<string, object> NewtonsoftExtensionData { get; set; }
+
+            [System.Text.Json.Serialization.JsonExtensionData()]
+            public Dictionary<string, object> NetExtensionData { get; set; }
+
             public complex(double d, string str, bool b, double[] da, simple s)
             {
                 this.dbl = d;
@@ -113,6 +119,8 @@ public complex(double d, string str, bool b, double[] da, simple s)
                 this.json = null;
                 this.id = Guid.NewGuid().ToString();
                 this.pk = "Test";
+                this.NetExtensionData = new Dictionary<string, object>();
+                this.NewtonsoftExtensionData = new Dictionary<string, object>();
             }
 
             public override string ToString()
@@ -236,7 +244,15 @@ public void ValidateSQLTranslation()
             inputs.Add(new LinqTestInput("Select new constructor", b => dataQuery(b).Select(x => new TimeSpan(x.x))));
             inputs.Add(new LinqTestInput("Select method id", b => dataQuery(b).Select(x => id(x))));
             inputs.Add(new LinqTestInput("Select identity", b => dataQuery(b).Select(x => x)));
-            inputs.Add(new LinqTestInput("Select simple property", b => dataQuery(b).Select(x => x.x)));            
+            inputs.Add(new LinqTestInput("Select simple property", b => dataQuery(b).Select(x => x.x)));
+            inputs.Add(new LinqTestInput("Select extension data", b => dataQuery(b).Select(x => new complex() {
+                NewtonsoftExtensionData = new() { 
+                    { "test", 1.5 }
+                },
+                NetExtensionData = new() { 
+                    { "OtherTest", 1.5 }
+                }
+            } )));
             this.ExecuteTestSuite(inputs);
         }
 
@@ -264,6 +280,10 @@ public void ValidateSQLTranslationComplexData()
                 obj.str = random.NextDouble() < 0.1 ? "5" : LinqTestsCommon.RandomString(random, random.Next(MaxStringLength));
                 obj.id = Guid.NewGuid().ToString();
                 obj.pk = "Test";
+                obj.NewtonsoftExtensionData = new Dictionary<string, object>() { 
+                    ["age"] = 32,
+                    ["tags"] = new [] { "item-1", "item-2" }
+                };
                 return obj;
             };
             Func<bool, IQueryable<complex>> getQuery = LinqTestsCommon.GenerateTestCosmosData<complex>(createDataObj, Records, testContainer);
@@ -285,6 +305,14 @@ public void ValidateSQLTranslationComplexData()
             inputs.Add(new LinqTestInput("SelectMany x -> Select y", b => getQuery(b).SelectMany(x => x.dblArray.Select(y => y))));
             inputs.Add(new LinqTestInput("SelectMany x -> Select x.y", b => getQuery(b).SelectMany(x => x.dblArray.Select(y => y))));
             inputs.Add(new LinqTestInput("SelectMany array", b => getQuery(b).SelectMany(x => x.dblArray)));
+
+            inputs.Add(new LinqTestInput("Select where extensiondata", b => getQuery(b).Where(p => (int)p.NewtonsoftExtensionData["age"] > 18).Select(x => new { Age = (int)x.NewtonsoftExtensionData["age"] })));
+            inputs.Add(new LinqTestInput("Select where extensiondata contains", b => getQuery(b).Where(p => ((string[])p.NewtonsoftExtensionData["tags"]).Contains("item-1")).Select(x => (string[])x.NewtonsoftExtensionData["tags"] )));
+
+            // TODO: SelectMany does not currently work with Dictionary objects, the snapshot represents
+            // the current (broken) behavior
+            inputs.Add(new LinqTestInput("SelectMany where extensiondata contains", b => getQuery(b).Where(p => ((string[])p.NewtonsoftExtensionData["tags"]).Contains("item-1")).SelectMany(x => (object[])x.NewtonsoftExtensionData["tags"] )));
+
             this.ExecuteTestSuite(inputs);
         }
 

diff --git a/...soft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Linq/CosmosLinqJsonConverterTests.cs b/...soft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Linq/CosmosLinqJsonConverterTests.cs
@@ -5,6 +5,7 @@
 namespace Microsoft.Azure.Cosmos.Linq
 {
     using System;
+    using System.Collections.Generic;
     using System.Globalization;
     using System.IO;
     using System.Linq;
@@ -152,6 +153,35 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
             }
         }
 
+        [TestMethod]
+        public void TestNewtonsoftExtensionDataQuery()
+        {
+            Expression<Func<DocumentWithExtensionData, bool>> expr = a => (string)a.NewtonsoftExtensionData["foo"] == "bar";
+            string sql = SqlTranslator.TranslateExpression(expr.Body);
+
+            Assert.AreEqual("(a[\"foo\"] = \"bar\")", sql);
+        }
+
+        [TestMethod]
+        public void TestSystemTextJsonExtensionDataQuery()
+        {
+            Expression<Func<DocumentWithExtensionData, bool>> expr = a => ((object)a.NetExtensionData["foo"]) == "bar";
+            string sql = SqlTranslator.TranslateExpression(expr.Body);
+
+            // TODO: This is a limitation in the translator. It should be able to handle STJ extension data, if a custom
+            // JSON serializer is specified.
+            Assert.AreEqual("(a[\"NetExtensionData\"][\"foo\"] = \"bar\")", sql);
+        }
+
+        class DocumentWithExtensionData
+        {
+            [Newtonsoft.Json.JsonExtensionData(ReadData = true, WriteData = true)]
+            public Dictionary<string, object> NewtonsoftExtensionData { get; set; }
+
+            [System.Text.Json.Serialization.JsonExtensionData()]
+            public Dictionary<string, System.Text.Json.JsonElement> NetExtensionData { get; set; }
+        }
+
         /// <remarks>
         // See: https://github.com/Azure/azure-cosmos-dotnet-v3/blob/master/Microsoft.Azure.Cosmos.Samples/Usage/SystemTextJson/CosmosSystemTextJsonSerializer.cs
         /// </remarks>