Add System.Text.Json to JSON serializer benchmarks (#2740)

* Add System.Text.Json to serializer benchmarks. * Extend source generator tests as a mode in the main JsonSerializer benchmarks. * Update src/benchmarks/micro/MicroBenchmarks.csproj Co-authored-by: Adam Sitnik <[email protected]> * Update src/benchmarks/micro/Serializers/Json_ToString.cs Co-authored-by: Adam Sitnik <[email protected]> Co-authored-by: Adam Sitnik <[email protected]>
dotnet · Nov 25, 2022 · 44e66a9 · 44e66a9
1 parent 0883707
commit 44e66a9
Show file tree

Hide file tree

Showing 9 changed files with 240 additions and 26 deletions.
diff --git a/src/benchmarks/micro/MicroBenchmarks.csproj b/src/benchmarks/micro/MicroBenchmarks.csproj
@@ -14,9 +14,8 @@
     <DebugType>portable</DebugType>
     <DebugSymbols>true</DebugSymbols>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <EnableXlfLocalization>false</EnableXlfLocalization>
-    <!-- this repo needs to build with netcoreapp3.1, so we can't use latest C# syntax -->
-    <LangVersion>8.0</LangVersion>
+    <EnableXlfLocalization>false</EnableXlfLocalization>    
+    <LangVersion>9.0</LangVersion>
 
     <!-- Allow building with one major version, and running using a sdk with a higher major version -->
     <RollForward Condition="'$(BuildingForWasm)' == 'true'">LatestMajor</RollForward>
@@ -27,6 +26,8 @@
         <PropertyGroup>
             <ExtensionsVersion>3.1.22</ExtensionsVersion>
             <SystemVersion>4.7.1</SystemVersion>
+            <!-- this repo needs to build with netcoreapp3.1, so we can't use latest C# syntax -->
+            <LangVersion>8.0</LangVersion>
         </PropertyGroup>
     </When>
     <When Condition="'$(TargetFramework)' == 'net6.0'">

diff --git a/src/benchmarks/micro/Serializers/DataGenerator.cs b/src/benchmarks/micro/Serializers/DataGenerator.cs
@@ -7,6 +7,11 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
+using System.Text.Json;
+#if NET6_0_OR_GREATER
+using System.Text.Json.Serialization;
+using System.Text.Json.Serialization.Metadata;
+#endif
 using System.Xml;
 using System.Xml.Serialization;
 using BenchmarkDotNet.Extensions;
@@ -215,6 +220,21 @@ private static XmlElement CreateXmlElement()
             xmlElement.InnerText = "Element innertext";
             return xmlElement;
         }
+
+        internal static JsonSerializerOptions GetJsonSerializerOptions(SystemTextJsonSerializationMode mode)
+            => mode switch
+            {
+                SystemTextJsonSerializationMode.Reflection => new JsonSerializerOptions(),
+#if NET6_0_OR_GREATER
+                SystemTextJsonSerializationMode.SourceGen => SystemTextJsonSourceGeneratedContext.Default.Options,
+#endif
+                _ => throw new NotSupportedException(mode.ToString())
+            };
+
+#if NET6_0_OR_GREATER
+        internal static JsonTypeInfo<T> GetSystemTextJsonSourceGenMetadata<T>()
+            => (JsonTypeInfo<T>)SystemTextJsonSourceGeneratedContext.Default.GetTypeInfo(typeof(T));
+#endif
     }
 
     // the view models come from a real world app called "AllReady"
@@ -421,4 +441,42 @@ public void WriteXml(XmlWriter writer)
             writer.WriteAttributeString("BoolValue", BoolValue.ToString());
         }
     }
+
+    public class ClassWithObjectProperty
+    {
+        public object Prop { get; set; }
+    }
+
+    public enum SystemTextJsonSerializationMode
+    {
+        Reflection = 0,
+        SourceGen = 1
+    }
+
+#if NET6_0_OR_GREATER
+    [JsonSerializable(typeof(ClassWithObjectProperty))]
+    [JsonSerializable(typeof(LoginViewModel))]
+    [JsonSerializable(typeof(Location))]
+    [JsonSerializable(typeof(IndexViewModel))]
+    [JsonSerializable(typeof(MyEventsListerViewModel))]
+    [JsonSerializable(typeof(BinaryData))]
+    [JsonSerializable(typeof(CollectionsOfPrimitives))]
+    [JsonSerializable(typeof(XmlElement))]
+    [JsonSerializable(typeof(SimpleStructWithProperties))]
+    [JsonSerializable(typeof(SimpleListOfInt))]
+    [JsonSerializable(typeof(ClassImplementingIXmlSerialiable))]
+    [JsonSerializable(typeof(Dictionary<string, string>))]
+    [JsonSerializable(typeof(ImmutableDictionary<string, string>))]
+    [JsonSerializable(typeof(ImmutableSortedDictionary<string, string>))]
+    [JsonSerializable(typeof(HashSet<string>))]
+    [JsonSerializable(typeof(ArrayList))]
+    [JsonSerializable(typeof(Hashtable))]
+    [JsonSerializable(typeof(LargeStructWithProperties))]
+    [JsonSerializable(typeof(DateTimeOffset?))]
+    [JsonSerializable(typeof(int))]
+    [JsonSerializable(typeof(object))]
+    internal partial class SystemTextJsonSourceGeneratedContext : JsonSerializerContext
+    {
+    }
+#endif
 }
diff --git a/src/benchmarks/micro/Serializers/Json_FromStream.cs b/src/benchmarks/micro/Serializers/Json_FromStream.cs
@@ -2,11 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using System.IO;
 using System.Runtime.Serialization.Json;
 using System.Text;
-using BenchmarkDotNet.Attributes;
-using BenchmarkDotNet.Attributes.Filters;
 
 namespace MicroBenchmarks.Serializers
 {
@@ -117,6 +117,49 @@ public T DataContractJsonSerializer_()
             return (T)dataContractJsonSerializer.ReadObject(memoryStream);
         }
 
+#if NET6_0_OR_GREATER
+        [GlobalSetup(Target = nameof(SystemTextJson_Reflection_))]
+        public void SetupSystemTextJson_Reflection_()
+        {
+            value = DataGenerator.Generate<T>();
+
+            // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
+            memoryStream.Position = 0;
+            System.Text.Json.JsonSerializer.Serialize(memoryStream, value);
+        }
+
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_Reflection")]
+        public T SystemTextJson_Reflection_()
+        {
+            memoryStream.Position = 0;
+            return System.Text.Json.JsonSerializer.Deserialize<T>(memoryStream);
+        }
+
+        private System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> sourceGenMetadata;
+
+        [GlobalSetup(Target = nameof(SystemTextJson_SourceGen_))]
+        public void SetupSystemTextJson_SourceGen_()
+        {
+            value = DataGenerator.Generate<T>();
+            sourceGenMetadata = DataGenerator.GetSystemTextJsonSourceGenMetadata<T>();
+
+            // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
+            memoryStream.Position = 0;
+            System.Text.Json.JsonSerializer.Serialize(memoryStream, value, sourceGenMetadata);
+        }
+
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_SourceGen")]
+        public T SystemTextJson_SourceGen_()
+        {
+            memoryStream.Position = 0;
+            return System.Text.Json.JsonSerializer.Deserialize(memoryStream, sourceGenMetadata);
+        }
+#endif
+
         [GlobalCleanup]
         public void Cleanup() => memoryStream.Dispose();
 

diff --git a/src/benchmarks/micro/Serializers/Json_FromString.cs b/src/benchmarks/micro/Serializers/Json_FromString.cs
@@ -41,5 +41,27 @@ public class Json_FromString<T>
         [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Utf8Json")]
         public T Utf8Json_() => Utf8Json.JsonSerializer.Deserialize<T>(serialized);
+
+        [GlobalSetup(Target = nameof(SystemTextJson_Reflection_))]
+        public void SetupSystemTextJson_Reflection_() => serialized = System.Text.Json.JsonSerializer.Serialize(DataGenerator.Generate<T>());
+
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_Reflection")]
+        public T SystemTextJson_Reflection_() => System.Text.Json.JsonSerializer.Deserialize<T>(serialized);
+
+#if NET6_0_OR_GREATER
+        private System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> sourceGenMetadata;
+
+        [GlobalSetup(Target = nameof(SystemTextJson_SourceGen_))]
+        public void SetupSystemTextJson_SourceGen_()
+        {
+            sourceGenMetadata = DataGenerator.GetSystemTextJsonSourceGenMetadata<T>();
+            serialized = System.Text.Json.JsonSerializer.Serialize(DataGenerator.Generate<T>(), sourceGenMetadata);
+        }
+
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_SourceGen")]
+        public T SystemTextJson_SourceGen_() => System.Text.Json.JsonSerializer.Deserialize(serialized, sourceGenMetadata);
+#endif
     }
 }
diff --git a/src/benchmarks/micro/Serializers/Json_ToStream.cs b/src/benchmarks/micro/Serializers/Json_ToStream.cs
@@ -72,10 +72,49 @@ public void DataContractJsonSerializer_()
             dataContractJsonSerializer.WriteObject(memoryStream, value);
         }
 
+#if NET6_0_OR_GREATER
+        [GlobalSetup(Target = nameof(SystemTextJson_Reflection_))]
+        public void SetupSystemTextJson_Reflection_()
+        {
+            value = DataGenerator.Generate<T>();
+
+            // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
+        }
+
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_Reflection")]
+        public void SystemTextJson_Reflection_()
+        {
+            memoryStream.Position = 0;
+            System.Text.Json.JsonSerializer.Serialize(memoryStream, value);
+        }
+
+        private System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> sourceGenMetadata;
+
+        [GlobalSetup(Target = nameof(SystemTextJson_SourceGen_))]
+        public void SetupSystemTextJson_SourceGen_()
+        {
+            value = DataGenerator.Generate<T>();
+            sourceGenMetadata = DataGenerator.GetSystemTextJsonSourceGenMetadata<T>();
+
+            // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
+        }
+
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_SourceGen")]
+        public void SystemTextJson_SourceGen_()
+        {
+            memoryStream.Position = 0;
+            System.Text.Json.JsonSerializer.Serialize(memoryStream, value, sourceGenMetadata);
+        }
+#endif
+
         [GlobalCleanup]
         public void Cleanup()
         {
-            streamWriter.Dispose();
+            streamWriter?.Dispose();
             memoryStream.Dispose();
         }
     }

diff --git a/src/benchmarks/micro/Serializers/Json_ToString.cs b/src/benchmarks/micro/Serializers/Json_ToString.cs
@@ -17,9 +17,18 @@ namespace MicroBenchmarks.Serializers
     public class Json_ToString<T>
     {
         private T value;
+#if NET6_0_OR_GREATER
+        private System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> sourceGenMetadata;
+#endif
 
         [GlobalSetup]
-        public void Setup() => value = DataGenerator.Generate<T>();
+        public void Setup()
+        {
+            value = DataGenerator.Generate<T>();
+#if NET6_0_OR_GREATER
+            sourceGenMetadata = DataGenerator.GetSystemTextJsonSourceGenMetadata<T>();
+#endif
+        }
 
         [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]
@@ -35,5 +44,15 @@ public class Json_ToString<T>
 
         // DataContractJsonSerializer does not provide an API to serialize to string
         // so it's not included here (apples vs apples thing)
+
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_Reflection")]
+        public string SystemTextJson_Reflection_() => System.Text.Json.JsonSerializer.Serialize(value);
+
+#if NET6_0_OR_GREATER
+        [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
+        [Benchmark(Description = "SystemTextJson_SourceGen")]
+        public string SystemTextJson_SourceGen_() => System.Text.Json.JsonSerializer.Serialize(value, sourceGenMetadata);
+#endif
     }
 }
diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ColdStartSerialization.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ColdStartSerialization.cs
@@ -40,6 +40,20 @@ private T RoundtripSerialization(JsonSerializerOptions options)
         [Benchmark]
         public T NewDefaultOptions() => RoundtripSerialization(new JsonSerializerOptions());
 
+#if NET6_0_OR_GREATER
+        [Benchmark]
+        public T CachedJsonSerializerContext() 
+            => RoundtripSerialization(SystemTextJsonSourceGeneratedContext.Default.Options);
+
+        [Benchmark]
+        public T NewJsonSerializerContext()
+        {
+            var options = new JsonSerializerOptions();
+            options.AddContext<SystemTextJsonSourceGeneratedContext>();
+            return RoundtripSerialization(options);
+        }
+#endif
+
         [Benchmark]
         public T NewCustomizedOptions() => 
             RoundtripSerialization(
@@ -61,11 +75,11 @@ public T NewCustomConverter() =>
 
         [Benchmark]
         public T NewCachedCustomConverter() =>
-            RoundtripSerialization(
-                new JsonSerializerOptions
-                {
-                    WriteIndented = true,
-                    Converters = { _converter }
-                });
+    RoundtripSerialization(
+        new JsonSerializerOptions
+        {
+            WriteIndented = true,
+            Converters = { _converter }
+        });
     }
 }
diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs
@@ -32,6 +32,14 @@ namespace System.Text.Json.Serialization.Tests
     [AotFilter("Currently not supported due to missing metadata.")]
     public class ReadJson<T>
     {
+#if NET6_0_OR_GREATER
+        [Params(SystemTextJsonSerializationMode.Reflection, SystemTextJsonSerializationMode.SourceGen)]
+#else
+        [Params(SystemTextJsonSerializationMode.Reflection)]
+#endif
+        public SystemTextJsonSerializationMode Mode;
+
+        private JsonSerializerOptions _options;
         private string _serialized;
         private byte[] _utf8Serialized;
         private MemoryStream _memoryStream;
@@ -40,37 +48,38 @@ public class ReadJson<T>
         public async Task Setup()
         {
             T value = DataGenerator.Generate<T>();
+            _options = DataGenerator.GetJsonSerializerOptions(Mode);
 
-            _serialized = JsonSerializer.Serialize(value);
+            _serialized = JsonSerializer.Serialize(value, _options);
 
             _utf8Serialized = Encoding.UTF8.GetBytes(_serialized);
 
             _memoryStream = new MemoryStream(capacity: short.MaxValue);
-            await JsonSerializer.SerializeAsync(_memoryStream, value);
+            await JsonSerializer.SerializeAsync(_memoryStream, value, _options);
         }
 
         [BenchmarkCategory(Categories.Libraries, Categories.JSON)]
         [Benchmark]
-        public T DeserializeFromString() => JsonSerializer.Deserialize<T>(_serialized);
+        public T DeserializeFromString() => JsonSerializer.Deserialize<T>(_serialized, _options);
 
         [BenchmarkCategory(Categories.Libraries, Categories.JSON)]
         [Benchmark]
-        public T DeserializeFromUtf8Bytes() => JsonSerializer.Deserialize<T>(_utf8Serialized);
+        public T DeserializeFromUtf8Bytes() => JsonSerializer.Deserialize<T>(_utf8Serialized, _options);
 
         [BenchmarkCategory(Categories.Libraries, Categories.JSON)]
         [Benchmark]
         public T DeserializeFromReader()
         {
             Utf8JsonReader reader = new Utf8JsonReader(_utf8Serialized);
-            return JsonSerializer.Deserialize<T>(ref reader);
+            return JsonSerializer.Deserialize<T>(ref reader, _options);
         }
 
         [BenchmarkCategory(Categories.Libraries, Categories.JSON, Categories.NoWASM)]
         [Benchmark]
         public async Task<T> DeserializeFromStream()
         {
             _memoryStream.Position = 0;
-            T value = await JsonSerializer.DeserializeAsync<T>(_memoryStream);
+            T value = await JsonSerializer.DeserializeAsync<T>(_memoryStream, _options);
             return value;
         }