From ad6c91b067ace7d8c25dc04bcb792ef6d48dbbb7 Mon Sep 17 00:00:00 2001 From: buyaa-n Date: Wed, 13 Nov 2019 16:14:46 -0800 Subject: [PATCH] Large json serialization/deserialization tests (dotnet/corefx#39668) * Test large json serializaiton/deserialization * Fixing test failure, applying feedback * Add circular dependency test with large object * Added test writing an object right at threshold level, removed unnecessary test cases * Fixing typo * Using VS generated POCO for test * Addressing feedback * Applying feedback * Applying feedback * Applying feedback * Refactored the test Json schema * Commenting failing case * Applying feedback * Applying feedback * Enabling fixed tests Commit migrated from https://github.com/dotnet/corefx/commit/20334125a70444b5e8cf35f3f661911cd85a78b5 --- .../Serialization/SampleTestData.OrderPayload.cs | 166 ++++++++++ .../tests/Serialization/Stream.WriteTests.cs | 364 +++++++++++++++++++++ .../tests/System.Text.Json.Tests.csproj | 1 + 3 files changed, 531 insertions(+) create mode 100644 src/libraries/System.Text.Json/tests/Serialization/SampleTestData.OrderPayload.cs diff --git a/src/libraries/System.Text.Json/tests/Serialization/SampleTestData.OrderPayload.cs b/src/libraries/System.Text.Json/tests/Serialization/SampleTestData.OrderPayload.cs new file mode 100644 index 0000000..6581844 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/Serialization/SampleTestData.OrderPayload.cs @@ -0,0 +1,166 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace System.Text.Json.Serialization.Tests.Schemas.OrderPayload +{ + public class Order + { + public long OrderNumber { get; set; } + public User Customer { get; set; } + public IEnumerable Products { get; set; } + public IEnumerable ShippingInfo { get; set; } + public bool OneTime { get; set; } + public bool Cancelled { get; set; } + public bool IsGift { get; set; } + public bool IsGPickUp { get; set; } + public Address ShippingAddress { get; set; } + public Address PickupAddress { get; set; } + public SampleEnumInt64 Coupon { get; set; } + public IEnumerable UserInteractions { get; set; } + public DateTime Created { get; set; } + public DateTime Updated { get; set; } + public DateTime Confirmed { get; set; } + public DateTime ShippingDate { get; set; } + public DateTime EstimatedDelivery { get; set; } + public IEnumerable RelatedOrder { get; set; } + public User ReviewedBy { get; set; } + } + + public class Product + { + public Guid ProductId { get; set; } + public string SKU { get; set; } + public TestClassWithInitializedProperties Brand { get; set; } + public SimpleTestClassWithNonGenericCollectionWrappers ProductCategory { get; set; } + public string Name { get; set; } + public string Description { get; set; } + public string Title { get; set; } + public Price Price { get; set; } + public bool BestChoice { get; set; } + public float AverageStars { get; set; } + public bool Featured { get; set; } + public TestClassWithInitializedProperties ProductRestrictions { get; set; } + public SimpleTestClassWithGenericCollectionWrappers SalesInfo { get; set; } + public IEnumerable Reviews { get; set; } + public SampleEnum Origin { get; set; } + public BasicCompany Manufacturer { get; set; } + public bool Fragile { get; set; } + public Uri DetailsUrl { get; set; } + public decimal NetWeight { get; set; } + public decimal GrossWeight { get; set; } + public int Length { get; set; } + public int Height { get; set; } + public int Width { get; set; } + public FeaturedImage FeaturedImage { get; set; } + public PreviewImage PreviewImage { get; set; } + public IEnumerable KeyWords; + public IEnumerable RelatedImages { get; set; } + public Uri RelatedVideo { get; set; } + public DateTime DeletedAt { get; set; } + public DateTime GuaranteeStartsAt { get; set; } + public DateTime GuaranteeEndsAt { get; set; } + public DateTime Created { get; set; } + public DateTime Updated { get; set; } + public bool IsActive { get; set; } + public IEnumerable SimilarProducts { get; set; } + public IEnumerable RelatedProducts { get; set; } + } + + public class Review + { + public long ReviewId { get; set; } + public User Customer { get; set; } + public string ProductSku { get; set; } + public string CustomerName { get; set; } + public int Stars { get; set; } + public string Title { get; set; } + public string Comment { get; set; } + public IEnumerable Images { get; set; } + } + + public class Comment + { + public long Id { get; set; } + public long OrderNumber { get; set; } + public User Customer { get; set; } + public User Employee { get; set; } + public IEnumerable Responses { get; set; } + public string Title { get; set; } + public string Message { get; set; } + } + + public class ShippingInfo + { + public long OrderNumber { get; set; } + public User Employee { get; set; } + public string CarrierId { get; set; } + public string ShippingType { get; set; } + public DateTime EstimatedDelivery { get; set; } + public Uri Tracking { get; set; } + public string CarrierName { get; set; } + public string HandlingInstruction { get; set; } + public string CurrentStatus { get; set; } + public bool IsDangerous { get; set; } + } + + public class Price + { + public Product Product { get; set; } + public bool AllowDiscount { get; set; } + public decimal OriginalPrice { get; set; } + public decimal RecommendedPrice { get; set; } + public decimal FinalPrice { get; set; } + public SampleEnumInt16 DiscountType { get; set; } + } + + public class PreviewImage + { + public string Id { get; set; } + public string Filter { get; set; } + public string Size { get; set; } + public int Width { get; set; } + public int Height { get; set; } + } + + public class FeaturedImage + { + public string Id { get; set; } + public int Width { get; set; } + public int Height { get; set; } + public string PhotoId { get; set; } + } + + public class Image + { + public string Id { get; set; } + public int Width { get; set; } + public int Height { get; set; } + } + + public class User + { + public BasicPerson PersonalInfo { get; set; } + public string UserId { get; set; } + public string Name { get; set; } + public string Username { get; set; } + public DateTime CreatedAt { get; set; } + public DateTime UpdatedAt { get; set; } + public string ImageId { get; set; } + public string TwitterId { get; set; } + public string FacebookId { get; set; } + public int SubscriptionType { get; set; } + public bool IsNew { get; set; } + public bool IsEmployee { get; set; } + public UserType UserType { get; set; } + } + + public enum UserType + { + Customer = 1, + Employee = 2, + Supplier = 3 + } +} diff --git a/src/libraries/System.Text.Json/tests/Serialization/Stream.WriteTests.cs b/src/libraries/System.Text.Json/tests/Serialization/Stream.WriteTests.cs index 27ca497..e19db69 100644 --- a/src/libraries/System.Text.Json/tests/Serialization/Stream.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/Serialization/Stream.WriteTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; +using System.Text.Json.Serialization.Tests.Schemas.OrderPayload; using System.Threading.Tasks; using Xunit; @@ -189,6 +190,8 @@ namespace System.Text.Json.Serialization.Tests [InlineData(4000)] [InlineData(8000)] [InlineData(16000)] + [InlineData(32000)] + [InlineData(64000)] public static async Task LargeJsonFile(int bufferSize) { const int SessionResponseCount = 100; @@ -258,6 +261,367 @@ namespace System.Text.Json.Serialization.Tests Assert.Equal(SessionResponseCount, deserializedList.Count); } } + + [Theory] + [InlineData(1, true, true)] + [InlineData(1, true, false)] + [InlineData(1, false, true)] + [InlineData(1, false, false)] + [InlineData(10, true, false)] + [InlineData(10, false, false)] + [InlineData(100, false, false)] + [InlineData(1000, false, false)] + public static async Task VeryLargeJsonFileTest(int payloadSize, bool ignoreNull, bool writeIndented) + { + List list = PopulateLargeObject(payloadSize); + + JsonSerializerOptions options = new JsonSerializerOptions + { + IgnoreNullValues = ignoreNull, + WriteIndented = writeIndented + }; + + string json = JsonSerializer.Serialize(list, options); + + // Sync case. + { + List deserializedList = JsonSerializer.Deserialize>(json, options); + Assert.Equal(payloadSize, deserializedList.Count); + + string jsonSerialized = JsonSerializer.Serialize(deserializedList, options); + Assert.Equal(json, jsonSerialized); + } + + // Async case. + using (var memoryStream = new MemoryStream()) + { + await JsonSerializer.SerializeAsync(memoryStream, list, options); + string jsonSerialized = Encoding.UTF8.GetString(memoryStream.ToArray()); + Assert.Equal(json, jsonSerialized); + + memoryStream.Position = 0; + List deserializedList = await JsonSerializer.DeserializeAsync>(memoryStream, options); + Assert.Equal(payloadSize, deserializedList.Count); + } + } + + [Theory] + [InlineData(1, true, true)] + [InlineData(1, true, false)] + [InlineData(1, false, true)] + [InlineData(1, false, false)] + [InlineData(2, true, false)] + [InlineData(2, false, false)] + [InlineData(4, false, false)] + [InlineData(8, false, false)] + [InlineData(16, false, false)] + public static async Task DeepNestedJsonFileTest(int depthFactor, bool ignoreNull, bool writeIndented) + { + int length = 10 * depthFactor; + List[] orders = new List[length]; + orders[0] = PopulateLargeObject(1); + for (int i = 1; i < length; i++ ) + { + orders[i] = PopulateLargeObject(1); + orders[i - 1][0].RelatedOrder = orders[i]; + } + + JsonSerializerOptions options = new JsonSerializerOptions() + { + MaxDepth = depthFactor * 64, + IgnoreNullValues = ignoreNull, + WriteIndented = writeIndented + }; + string json = JsonSerializer.Serialize(orders[0], options); + + // Sync case. + { + List deserializedList = JsonSerializer.Deserialize>(json, options); + + string jsonSerialized = JsonSerializer.Serialize(deserializedList, options); + Assert.Equal(json, jsonSerialized); + } + + // Async case. + using (var memoryStream = new MemoryStream()) + { + await JsonSerializer.SerializeAsync(memoryStream, orders[0], options); + string jsonSerialized = Encoding.UTF8.GetString(memoryStream.ToArray()); + Assert.Equal(json, jsonSerialized); + + memoryStream.Position = 0; + List deserializedList = await JsonSerializer.DeserializeAsync>(memoryStream, options); + } + } + + [Theory] + [InlineData(1)] + [InlineData(16)] + public static async Task DeepNestedJsonFileCircularDependencyTest(int depthFactor) + { + int length = 10 * depthFactor; + List[] orders = new List[length]; + orders[0] = PopulateLargeObject(1000); + for (int i = 1; i < length; i++) + { + orders[i] = PopulateLargeObject(1); + orders[i - 1][0].RelatedOrder = orders[i]; + } + orders[length - 1][0].RelatedOrder = orders[0]; + + JsonSerializerOptions options = new JsonSerializerOptions() + { + MaxDepth = depthFactor * 64, + IgnoreNullValues = true + }; + + Assert.Throws (() => JsonSerializer.Serialize(orders[0], options)); + + using (var memoryStream = new MemoryStream()) + { + await Assert.ThrowsAsync(async () => await JsonSerializer.SerializeAsync(memoryStream, orders[0], options)); + } + } + + [Theory] + [InlineData(128)] + [InlineData(1024)] + [InlineData(4096)] + [InlineData(8192)] + [InlineData(16384)] + [InlineData(65536)] + public static async void FlushThresholdTest(int bufferSize) + { + // bufferSize * 0.9 is the threshold size from codebase, subtract 2 for [" characters, then create a + // string containing (threshold - 2) amount of char 'a' which when written into output buffer produces buffer + // which size equal to or very close to threshold size, then adding the string to the list, then adding a big + // object to the list which changes depth of written json and should cause buffer flush + int thresholdSize = (int)(bufferSize * 0.9 - 2); + FlushThresholdTestClass serializeObject = new FlushThresholdTestClass(GenerateListOfSize(bufferSize)); + List list = new List(); + string stringOfThresholdSize = new string('a', thresholdSize); + list.Add(stringOfThresholdSize); + serializeObject.StringProperty = stringOfThresholdSize; + list.Add(serializeObject); + JsonSerializerOptions options = new JsonSerializerOptions(); + options.DefaultBufferSize = bufferSize; + + string json = JsonSerializer.Serialize(list); + + using (var memoryStream = new MemoryStream()) + { + await JsonSerializer.SerializeAsync(memoryStream, list, options); + string jsonSerialized = Encoding.UTF8.GetString(memoryStream.ToArray()); + Assert.Equal(json, jsonSerialized); + + List deserializedList = JsonSerializer.Deserialize>(json, options); + Assert.Equal(stringOfThresholdSize, ((JsonElement)deserializedList[0]).GetString()); + JsonElement obj = (JsonElement)deserializedList[1]; + Assert.Equal(stringOfThresholdSize, obj.GetProperty("StringProperty").GetString()); + } + } + + private class FlushThresholdTestClass + { + public string StringProperty { get; set; } + public List ListOfInts { get; set; } + public FlushThresholdTestClass(List list) + { + ListOfInts = list; + } + } + + private static List GenerateListOfSize(int size) + { + List list = new List(); + for (int i = 0; i < size; i++) + { + list.Add(1); + } + return list; + } + + private static List PopulateLargeObject(int size) + { + List orders = new List(size); + for (int i = 0; i < size; i++) + { + Order order = new Order + { + OrderNumber = i, + Customer = new User + { + UserId = "222ffbbb888kkk", + Name = "John Doe", + Username = "johndoe", + CreatedAt = new DateTime(), + ImageId = string.Empty, + UserType = UserType.Customer, + UpdatedAt = new DateTime(), + TwitterId = string.Empty, + FacebookId = "9988998877662222111", + SubscriptionType = 2, + IsNew = true, + IsEmployee = false + }, + ShippingInfo = new List + { + new ShippingInfo() + { + OrderNumber = i, + Employee = new User + { + UserId = "222ffbbb888" + i, + Name = "Shipping Coordinator", + Username = "coordinator" + i, + CreatedAt = new DateTime(), + ImageId = string.Empty, + UserType = UserType.Employee, + UpdatedAt = new DateTime(), + TwitterId = string.Empty, + SubscriptionType = 0, + IsEmployee = true + }, + CarrierId = "TTT123999MMM", + ShippingType = "Ground", + EstimatedDelivery = new DateTime(), + Tracking = new Uri("http://TestShipCompany.test/track/123" + i), + CarrierName = "TestShipCompany", + HandlingInstruction = "Do cats eat bats? Do cats eat bats. Do cats eat bats? Do cats eat bats. Do cats eat bats? Do cats eat bats. Do cats eat bats? Do cats eat bats", + CurrentStatus = "Out for delivery", + IsDangerous = false + } + }, + OneTime = true, + Cancelled = false, + IsGift = i % 2 == 0, + IsGPickUp = i % 5 == 0, + ShippingAddress = new Address() + { + City = "Redmond" + }, + PickupAddress = new Address + { + City = "Bellevue" + }, + Coupon = SampleEnumInt64.Max, + UserInteractions = new List + { + new Comment + { + Id = 200 + i, + OrderNumber = i, + Customer = new User + { + UserId = "222ffbbb888kkk", + Name = "John Doe", + Username = "johndoe", + CreatedAt = new DateTime(), + ImageId = string.Empty, + UserType = UserType.Customer, + UpdatedAt = new DateTime(), + TwitterId = "twitterId" + i, + FacebookId = "9988998877662222111", + SubscriptionType = 2, + IsNew = true, + IsEmployee = false + }, + Title = "Green Field", + Message = "Down, down, down. Would the fall never come to an end! ‘I wonder how many miles I’ve fallen by this time. I think—’ (for, you see, Alice had learnt several things of this sort in her lessons in the schoolroom, and though this was not a very good opportunity for showing off her knowledge, as there was no one to listen to her, still it was good practice to say it over) ‘—yes, that’s about the right distance—but then I wonder what Latitude or Longitude I’ve got to", + Responses = new List() + } + }, + Created = new DateTime(2019, 11, 10), + Confirmed = new DateTime(2019, 11, 11), + ShippingDate = new DateTime(2019, 11, 12), + EstimatedDelivery = new DateTime(2019, 11, 15), + ReviewedBy = new User() + { + UserId = "222ffbbb888" + i, + Name = "Shipping Coordinator", + Username = "coordinator" + i, + CreatedAt = new DateTime(), + ImageId = string.Empty, + UserType = UserType.Employee, + UpdatedAt = new DateTime(), + TwitterId = string.Empty, + SubscriptionType = 0, + IsEmployee = true + } + }; + List products = new List(); + for (int j = 0; j < i % 4; j++) + { + Product product = new Product() + { + ProductId = Guid.NewGuid(), + Name = "Surface Pro", + SKU = "LL123" + j, + Brand = new TestClassWithInitializedProperties(), + ProductCategory = new SimpleTestClassWithNonGenericCollectionWrappers(), + Description = "Down, down, down. Would the fall never come to an end! ‘I wonder how many miles I’ve fallen by this time. I think—’ (for, you see, Alice had learnt several things of this sort in her lessons in the schoolroom, and though this was not a very good opportunity for showing off her knowledge, as there was no one to listen to her, still it was good practice to say it over) ‘—yes, that’s about the right distance—but then I wonder what Latitude or Longitude I’ve got to", + Created = new DateTime(2000, 10, 12), + Title = "Surface Pro 6 for Business - 512GB", + Price = new Price(), + BestChoice = true, + AverageStars = 4.8f, + Featured = true, + ProductRestrictions = new TestClassWithInitializedProperties(), + SalesInfo = new SimpleTestClassWithGenericCollectionWrappers(), + Origin = SampleEnum.One, + Manufacturer = new BasicCompany(), + Fragile = true, + DetailsUrl = new Uri("http://dotnet.test/link/entries/entry/1"), + NetWeight = 2.7m, + GrossWeight = 3.3m, + Length = i, + Height = i + 1, + Width = i + 2, + FeaturedImage = new FeaturedImage(), + PreviewImage = new PreviewImage(), + KeyWords = new List { "surface", "pro", "laptop" }, + RelatedImages = new List(), + RelatedVideo = new Uri("http://dotnet.test/link/entries/entry/2"), + GuaranteeStartsAt = new DateTime(), + GuaranteeEndsAt = new DateTime(), + IsActive = true, + RelatedProducts = new List() + }; + product.SalesInfo.Initialize(); + List reviews = new List(); + for (int k = 0; k < i % 3; k++) + { + + Review review = new Review + { + Customer = new User + { + UserId = "333344445555", + Name = "Customer" + i + k, + Username = "cust" + i + k, + CreatedAt = new DateTime(), + ImageId = string.Empty, + UserType = UserType.Customer, + SubscriptionType = k + }, + ProductSku = product.SKU, + CustomerName = "Customer" + i + k, + Stars = j + k, + Title = $"Title {i}{j}{k}", + Comment = "", + Images = new List{ new Uri($"http://dotnet.test/link/images/image/{k}"), new Uri($"http://dotnet.test/link/images/image/{j}")}, + ReviewId = i + j +k + }; + reviews.Add(review); + } + product.Reviews = reviews; + products.Add(product); + } + order.Products = products; + orders.Add(order); + } + return orders; + } } public sealed class TestStream : Stream diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj index e9a5146..33bfb98 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj @@ -74,6 +74,7 @@ + -- 2.7.4