Large json serialization/deserialization tests (dotnet/corefx#39668)
authorbuyaa-n <bunamnan@microsoft.com>
Thu, 14 Nov 2019 00:14:46 +0000 (16:14 -0800)
committerAhson Khan <ahson_ahmedk@yahoo.com>
Thu, 14 Nov 2019 00:14:45 +0000 (16:14 -0800)
* Test large json serializaiton/deserialization

* Fixing test failure, applying feedback

* Add circular dependency test with large object

* Added test writing an object right at threshold level, removed unnecessary test cases

* Fixing typo

* Using VS generated POCO for test

* Addressing feedback

* Applying feedback

* Applying feedback

* Applying feedback

* Refactored the test Json schema

* Commenting failing case

* Applying feedback

* Applying feedback

* Enabling fixed tests

Commit migrated from https://github.com/dotnet/corefx/commit/20334125a70444b5e8cf35f3f661911cd85a78b5

src/libraries/System.Text.Json/tests/Serialization/SampleTestData.OrderPayload.cs [new file with mode: 0644]
src/libraries/System.Text.Json/tests/Serialization/Stream.WriteTests.cs
src/libraries/System.Text.Json/tests/System.Text.Json.Tests.csproj

diff --git a/src/libraries/System.Text.Json/tests/Serialization/SampleTestData.OrderPayload.cs b/src/libraries/System.Text.Json/tests/Serialization/SampleTestData.OrderPayload.cs
new file mode 100644 (file)
index 0000000..6581844
--- /dev/null
@@ -0,0 +1,166 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace System.Text.Json.Serialization.Tests.Schemas.OrderPayload
+{ 
+    public class Order
+    {
+        public long OrderNumber { get; set; }
+        public User Customer { get; set; }
+        public IEnumerable<Product> Products { get; set; }
+        public IEnumerable<ShippingInfo> ShippingInfo { get; set; }
+        public bool OneTime { get; set; }
+        public bool Cancelled { get; set; }
+        public bool IsGift { get; set; }
+        public bool IsGPickUp { get; set; }
+        public Address ShippingAddress { get; set; }
+        public Address PickupAddress { get; set; }
+        public SampleEnumInt64 Coupon { get; set; }
+        public IEnumerable<Comment> UserInteractions { get; set; }
+        public DateTime Created { get; set; }
+        public DateTime Updated { get; set; }
+        public DateTime Confirmed { get; set; }
+        public DateTime ShippingDate { get; set; }
+        public DateTime EstimatedDelivery { get; set; }
+        public IEnumerable<Order> RelatedOrder { get; set; }
+        public User ReviewedBy { get; set; }
+    }
+    
+    public class Product
+    {
+        public Guid ProductId { get; set; }
+        public string SKU { get; set; }
+        public TestClassWithInitializedProperties Brand { get; set; }
+        public SimpleTestClassWithNonGenericCollectionWrappers ProductCategory { get; set; }
+        public string Name { get; set; }
+        public string Description { get; set; }
+        public string Title { get; set; }
+        public Price Price { get; set; }
+        public bool BestChoice { get; set; }
+        public float AverageStars { get; set; }  
+        public bool Featured { get; set; }
+        public TestClassWithInitializedProperties ProductRestrictions { get; set; }
+        public SimpleTestClassWithGenericCollectionWrappers SalesInfo { get; set; }
+        public IEnumerable<Review> Reviews { get; set; }
+        public SampleEnum Origin { get; set; }
+        public BasicCompany Manufacturer { get; set; }
+        public bool Fragile { get; set; }
+        public Uri DetailsUrl { get; set; }
+        public decimal NetWeight { get; set; }
+        public decimal GrossWeight { get; set; }
+        public int Length { get; set; }
+        public int Height { get; set; }
+        public int Width { get; set; }
+        public FeaturedImage FeaturedImage { get; set; }
+        public PreviewImage PreviewImage { get; set; }
+        public IEnumerable<string> KeyWords;
+        public IEnumerable<Image> RelatedImages { get; set; }
+        public Uri RelatedVideo { get; set; }
+        public DateTime DeletedAt { get; set; }
+        public DateTime GuaranteeStartsAt { get; set; }
+        public DateTime GuaranteeEndsAt { get; set; }
+        public DateTime Created { get; set; }
+        public DateTime Updated { get; set; }
+        public bool IsActive { get; set; }
+        public IEnumerable<Product> SimilarProducts { get; set; }
+        public IEnumerable<Product> RelatedProducts { get; set; }
+    }
+
+    public class Review
+    {
+        public long ReviewId { get; set; } 
+        public User Customer { get; set; }
+        public string ProductSku { get; set; }
+        public string CustomerName { get; set; }
+        public int Stars { get; set; }
+        public string Title { get; set; }
+        public string Comment { get; set; }
+        public IEnumerable<Uri> Images { get; set; }
+    }
+
+    public class Comment
+    {
+        public long Id { get; set; }
+        public long OrderNumber { get; set; }
+        public User Customer { get; set; }
+        public User Employee { get; set; }
+        public IEnumerable<Comment> Responses { get; set; }
+        public string Title { get; set; }
+        public string Message { get; set; }
+    }
+
+    public class ShippingInfo
+    {
+        public long OrderNumber { get; set; }
+        public User Employee { get; set; }
+        public string CarrierId { get; set; }
+        public string ShippingType { get; set; }
+        public DateTime EstimatedDelivery { get; set; }
+        public Uri Tracking { get; set; }
+        public string CarrierName { get; set; }
+        public string HandlingInstruction { get; set; }
+        public string CurrentStatus { get; set; }
+        public bool IsDangerous { get; set; }
+    }
+
+    public class Price
+    {
+        public Product Product { get; set; }
+        public bool AllowDiscount { get; set; }
+        public decimal OriginalPrice { get; set; }
+        public decimal RecommendedPrice { get; set; }
+        public decimal FinalPrice { get; set; }
+        public SampleEnumInt16 DiscountType { get; set; }
+    }
+
+    public class PreviewImage
+    {
+        public string Id { get; set; }
+        public string Filter { get; set; }
+        public string Size { get; set; }
+        public int Width { get; set; }
+        public int Height { get; set; }
+    }
+
+    public class FeaturedImage
+    {
+        public string Id { get; set; }
+        public int Width { get; set; }
+        public int Height { get; set; }
+        public string PhotoId { get; set; }
+    }
+
+    public class Image
+    {
+        public string Id { get; set; }
+        public int Width { get; set; }
+        public int Height { get; set; }
+    }
+
+    public class User
+    {
+        public BasicPerson PersonalInfo { get; set; }
+        public string UserId { get; set; }
+        public string Name { get; set; }
+        public string Username { get; set; }
+        public DateTime CreatedAt { get; set; }
+        public DateTime UpdatedAt { get; set; }
+        public string ImageId { get; set; }
+        public string TwitterId { get; set; }
+        public string FacebookId { get; set; }
+        public int SubscriptionType { get; set; }
+        public bool IsNew { get; set; }
+        public bool IsEmployee { get; set; }
+        public UserType UserType { get; set; }
+    }
+
+    public enum UserType
+    {
+        Customer = 1,
+        Employee = 2,
+        Supplier = 3
+    }
+}
index 27ca497..e19db69 100644 (file)
@@ -4,6 +4,7 @@
 
 using System.Collections.Generic;
 using System.IO;
+using System.Text.Json.Serialization.Tests.Schemas.OrderPayload;
 using System.Threading.Tasks;
 using Xunit;
 
@@ -189,6 +190,8 @@ namespace System.Text.Json.Serialization.Tests
         [InlineData(4000)]
         [InlineData(8000)]
         [InlineData(16000)]
+        [InlineData(32000)]
+        [InlineData(64000)]
         public static async Task LargeJsonFile(int bufferSize)
         {
             const int SessionResponseCount = 100;
@@ -258,6 +261,367 @@ namespace System.Text.Json.Serialization.Tests
                 Assert.Equal(SessionResponseCount, deserializedList.Count);
             }
         }
+
+        [Theory]
+        [InlineData(1, true, true)]
+        [InlineData(1, true, false)]
+        [InlineData(1, false, true)]
+        [InlineData(1, false, false)]
+        [InlineData(10, true, false)]
+        [InlineData(10, false, false)]
+        [InlineData(100, false, false)]
+        [InlineData(1000, false, false)]
+        public static async Task VeryLargeJsonFileTest(int payloadSize, bool ignoreNull, bool writeIndented)
+        {
+            List<Order> list = PopulateLargeObject(payloadSize);
+
+            JsonSerializerOptions options = new JsonSerializerOptions
+            {
+                IgnoreNullValues = ignoreNull,
+                WriteIndented = writeIndented
+            };
+
+            string json = JsonSerializer.Serialize(list, options);
+
+            // Sync case.
+            {
+                List<Order> deserializedList = JsonSerializer.Deserialize<List<Order>>(json, options);
+                Assert.Equal(payloadSize, deserializedList.Count);
+
+                string jsonSerialized = JsonSerializer.Serialize(deserializedList, options);
+                Assert.Equal(json, jsonSerialized);
+            }
+
+            // Async case.
+            using (var memoryStream = new MemoryStream())
+            {
+                await JsonSerializer.SerializeAsync(memoryStream, list, options);
+                string jsonSerialized = Encoding.UTF8.GetString(memoryStream.ToArray());
+                Assert.Equal(json, jsonSerialized);
+
+                memoryStream.Position = 0;
+                List<Order> deserializedList = await JsonSerializer.DeserializeAsync<List<Order>>(memoryStream, options);
+                Assert.Equal(payloadSize, deserializedList.Count);
+            }
+        }
+
+        [Theory]
+        [InlineData(1, true, true)]
+        [InlineData(1, true, false)]
+        [InlineData(1, false, true)]
+        [InlineData(1, false, false)]
+        [InlineData(2, true, false)]
+        [InlineData(2, false, false)]
+        [InlineData(4, false, false)]
+        [InlineData(8, false, false)]
+        [InlineData(16, false, false)]
+        public static async Task DeepNestedJsonFileTest(int depthFactor, bool ignoreNull, bool writeIndented)
+        {
+            int length = 10 * depthFactor;
+            List<Order>[] orders = new List<Order>[length];
+            orders[0] = PopulateLargeObject(1);
+            for (int i = 1; i < length; i++ )
+            {
+                orders[i] = PopulateLargeObject(1);
+                orders[i - 1][0].RelatedOrder = orders[i];
+            }
+
+            JsonSerializerOptions options = new JsonSerializerOptions()
+            {
+                MaxDepth = depthFactor * 64,
+                IgnoreNullValues = ignoreNull,
+                WriteIndented = writeIndented
+            };
+            string json = JsonSerializer.Serialize(orders[0], options);
+
+            // Sync case.
+            {
+                List<Order> deserializedList = JsonSerializer.Deserialize<List<Order>>(json, options);
+
+                string jsonSerialized = JsonSerializer.Serialize(deserializedList, options);
+                Assert.Equal(json, jsonSerialized);
+            }
+
+            // Async case.
+            using (var memoryStream = new MemoryStream())
+            {
+                await JsonSerializer.SerializeAsync(memoryStream, orders[0], options);
+                string jsonSerialized = Encoding.UTF8.GetString(memoryStream.ToArray());
+                Assert.Equal(json, jsonSerialized);
+
+                memoryStream.Position = 0;
+                List<Order> deserializedList = await JsonSerializer.DeserializeAsync<List<Order>>(memoryStream, options);
+            }
+        }
+
+        [Theory]
+        [InlineData(1)]
+        [InlineData(16)]
+        public static async Task DeepNestedJsonFileCircularDependencyTest(int depthFactor)
+        {
+            int length = 10 * depthFactor;
+            List<Order>[] orders = new List<Order>[length];
+            orders[0] = PopulateLargeObject(1000);
+            for (int i = 1; i < length; i++)
+            {
+                orders[i] = PopulateLargeObject(1);
+                orders[i - 1][0].RelatedOrder = orders[i];
+            }
+            orders[length - 1][0].RelatedOrder = orders[0];
+
+            JsonSerializerOptions options = new JsonSerializerOptions()
+            {
+                MaxDepth = depthFactor * 64,
+                IgnoreNullValues = true
+            };
+
+            Assert.Throws<JsonException> (() => JsonSerializer.Serialize(orders[0], options));
+
+            using (var memoryStream = new MemoryStream())
+            {
+                await Assert.ThrowsAsync<JsonException>(async () => await JsonSerializer.SerializeAsync(memoryStream, orders[0], options));
+            }
+        }
+
+        [Theory]
+        [InlineData(128)]
+        [InlineData(1024)]
+        [InlineData(4096)]
+        [InlineData(8192)]
+        [InlineData(16384)]
+        [InlineData(65536)]
+        public static async void FlushThresholdTest(int bufferSize)
+        {
+            // bufferSize * 0.9 is the threshold size from codebase, subtract 2 for [" characters, then create a 
+            // string containing (threshold - 2) amount of char 'a' which when written into output buffer produces buffer 
+            // which size equal to or very close to threshold size, then adding the string to the list, then adding a big 
+            // object to the list which changes depth of written json and should cause buffer flush
+            int thresholdSize = (int)(bufferSize * 0.9 - 2);
+            FlushThresholdTestClass serializeObject = new FlushThresholdTestClass(GenerateListOfSize(bufferSize));
+            List<object> list = new List<object>();
+            string stringOfThresholdSize = new string('a', thresholdSize);
+            list.Add(stringOfThresholdSize);
+            serializeObject.StringProperty = stringOfThresholdSize;
+            list.Add(serializeObject);
+            JsonSerializerOptions options = new JsonSerializerOptions();
+            options.DefaultBufferSize = bufferSize;
+
+            string json = JsonSerializer.Serialize(list);
+
+            using (var memoryStream = new MemoryStream())
+            {
+                await JsonSerializer.SerializeAsync(memoryStream, list, options);
+                string jsonSerialized = Encoding.UTF8.GetString(memoryStream.ToArray());
+                Assert.Equal(json, jsonSerialized);
+
+                List<object> deserializedList = JsonSerializer.Deserialize<List<object>>(json, options);
+                Assert.Equal(stringOfThresholdSize, ((JsonElement)deserializedList[0]).GetString());
+                JsonElement obj = (JsonElement)deserializedList[1];             
+                Assert.Equal(stringOfThresholdSize, obj.GetProperty("StringProperty").GetString());
+            }
+        }
+
+        private class FlushThresholdTestClass 
+        {
+            public string StringProperty { get; set; }
+            public List<int> ListOfInts { get; set; }
+            public FlushThresholdTestClass(List<int> list)
+            {
+                ListOfInts = list;
+            }
+        }
+
+        private static List<int> GenerateListOfSize(int size)
+        {
+            List<int> list = new List<int>();
+            for (int i = 0; i < size; i++)
+            {
+                list.Add(1);
+            }
+            return list;
+        }
+
+        private static List<Order> PopulateLargeObject(int size)
+        {
+            List<Order> orders = new List<Order>(size);
+            for (int i = 0; i < size; i++)
+            {
+                Order order = new Order
+                {
+                    OrderNumber = i,
+                    Customer = new User
+                    {
+                        UserId = "222ffbbb888kkk",
+                        Name = "John Doe",
+                        Username = "johndoe",
+                        CreatedAt = new DateTime(),
+                        ImageId = string.Empty,
+                        UserType = UserType.Customer,
+                        UpdatedAt = new DateTime(),
+                        TwitterId = string.Empty,
+                        FacebookId = "9988998877662222111",
+                        SubscriptionType = 2,
+                        IsNew = true,
+                        IsEmployee = false
+                    },
+                    ShippingInfo = new List<ShippingInfo>
+                    {
+                        new ShippingInfo()
+                        {
+                            OrderNumber = i,
+                            Employee = new User
+                            {
+                                UserId = "222ffbbb888" + i,
+                                Name = "Shipping Coordinator",
+                                Username = "coordinator" + i,
+                                CreatedAt = new DateTime(),
+                                ImageId = string.Empty,
+                                UserType = UserType.Employee,
+                                UpdatedAt = new DateTime(),
+                                TwitterId = string.Empty,
+                                SubscriptionType = 0,
+                                IsEmployee = true
+                            },
+                            CarrierId = "TTT123999MMM",
+                            ShippingType = "Ground",
+                            EstimatedDelivery = new DateTime(),
+                            Tracking = new Uri("http://TestShipCompany.test/track/123" + i),
+                            CarrierName = "TestShipCompany",
+                            HandlingInstruction = "Do cats eat bats? Do cats eat bats. Do cats eat bats? Do cats eat bats. Do cats eat bats? Do cats eat bats. Do cats eat bats? Do cats eat bats",
+                            CurrentStatus = "Out for delivery",
+                            IsDangerous = false
+                        }
+                    },
+                    OneTime = true,
+                    Cancelled = false,
+                    IsGift = i % 2 == 0,
+                    IsGPickUp = i % 5 == 0,
+                    ShippingAddress = new Address()
+                    {
+                        City = "Redmond"
+                    },
+                    PickupAddress = new Address
+                    {
+                        City = "Bellevue"
+                    },
+                    Coupon = SampleEnumInt64.Max,
+                    UserInteractions = new List<Comment>
+                    {
+                        new Comment
+                        {
+                            Id = 200 + i,
+                            OrderNumber = i,
+                            Customer = new User
+                            {
+                                UserId = "222ffbbb888kkk",
+                                Name = "John Doe",
+                                Username = "johndoe",
+                                CreatedAt = new DateTime(),
+                                ImageId = string.Empty,
+                                UserType = UserType.Customer,
+                                UpdatedAt = new DateTime(),
+                                TwitterId = "twitterId" + i,
+                                FacebookId = "9988998877662222111",
+                                SubscriptionType = 2,
+                                IsNew = true,
+                                IsEmployee = false
+                            },
+                            Title = "Green Field",
+                            Message = "Down, down, down. Would the fall never come to an end! ‘I wonder how many miles I’ve fallen by this time. I think—’ (for, you see, Alice had learnt several things of this sort in her lessons in the schoolroom, and though this was not a very good opportunity for showing off her knowledge, as there was no one to listen to her, still it was good practice to say it over) ‘—yes, that’s about the right distance—but then I wonder what Latitude or Longitude I’ve got to",
+                            Responses = new List<Comment>()
+                        }
+                    },
+                    Created = new DateTime(2019, 11, 10),
+                    Confirmed = new DateTime(2019, 11, 11),
+                    ShippingDate = new DateTime(2019, 11, 12),
+                    EstimatedDelivery = new DateTime(2019, 11, 15),
+                    ReviewedBy = new User()
+                    {
+                        UserId = "222ffbbb888" + i,
+                        Name = "Shipping Coordinator",
+                        Username = "coordinator" + i,
+                        CreatedAt = new DateTime(),
+                        ImageId = string.Empty,
+                        UserType = UserType.Employee,
+                        UpdatedAt = new DateTime(),
+                        TwitterId = string.Empty,
+                        SubscriptionType = 0,
+                        IsEmployee = true
+                    }
+                };
+                List<Product> products = new List<Product>();
+                for (int j = 0; j < i % 4; j++)
+                {
+                    Product product = new Product()
+                    {
+                        ProductId = Guid.NewGuid(),
+                        Name = "Surface Pro",
+                        SKU = "LL123" + j,
+                        Brand = new TestClassWithInitializedProperties(),
+                        ProductCategory = new SimpleTestClassWithNonGenericCollectionWrappers(),
+                        Description = "Down, down, down. Would the fall never come to an end! ‘I wonder how many miles I’ve fallen by this time. I think—’ (for, you see, Alice had learnt several things of this sort in her lessons in the schoolroom, and though this was not a very good opportunity for showing off her knowledge, as there was no one to listen to her, still it was good practice to say it over) ‘—yes, that’s about the right distance—but then I wonder what Latitude or Longitude I’ve got to",
+                        Created = new DateTime(2000, 10, 12),
+                        Title = "Surface Pro 6 for Business - 512GB",
+                        Price = new Price(),
+                        BestChoice = true,
+                        AverageStars = 4.8f,
+                        Featured = true,
+                        ProductRestrictions = new TestClassWithInitializedProperties(),
+                        SalesInfo = new SimpleTestClassWithGenericCollectionWrappers(),
+                        Origin = SampleEnum.One,
+                        Manufacturer = new BasicCompany(),
+                        Fragile = true,
+                        DetailsUrl = new Uri("http://dotnet.test/link/entries/entry/1"),
+                        NetWeight = 2.7m,
+                        GrossWeight = 3.3m,
+                        Length = i,
+                        Height = i + 1,
+                        Width = i + 2,
+                        FeaturedImage = new FeaturedImage(),
+                        PreviewImage = new PreviewImage(),
+                        KeyWords = new List<string> { "surface", "pro", "laptop" },
+                        RelatedImages = new List<Image>(),
+                        RelatedVideo = new Uri("http://dotnet.test/link/entries/entry/2"),
+                        GuaranteeStartsAt = new DateTime(),
+                        GuaranteeEndsAt = new DateTime(),
+                        IsActive = true,
+                        RelatedProducts = new List<Product>()
+                    };
+                    product.SalesInfo.Initialize();
+                    List<Review> reviews = new List<Review>();
+                    for (int k = 0; k < i % 3; k++)
+                    {
+
+                        Review review = new Review
+                        {
+                            Customer = new User
+                            {
+                                UserId = "333344445555",
+                                Name = "Customer" + i + k,
+                                Username = "cust" + i + k,
+                                CreatedAt = new DateTime(),
+                                ImageId = string.Empty,
+                                UserType = UserType.Customer,
+                                SubscriptionType = k
+                            },
+                            ProductSku = product.SKU,
+                            CustomerName = "Customer" + i + k,
+                            Stars = j + k,
+                            Title = $"Title {i}{j}{k}",
+                            Comment = "",
+                            Images = new List<Uri>{ new Uri($"http://dotnet.test/link/images/image/{k}"), new Uri($"http://dotnet.test/link/images/image/{j}")},
+                            ReviewId = i + j +k
+                        };
+                        reviews.Add(review);
+                    }
+                    product.Reviews = reviews;
+                    products.Add(product);
+                }
+                order.Products = products;
+                orders.Add(order);
+            }
+            return orders;
+        }
     }
 
     public sealed class TestStream : Stream
index e9a5146..33bfb98 100644 (file)
@@ -74,6 +74,7 @@
     <Compile Include="Serialization\PropertyNameTests.cs" />
     <Compile Include="Serialization\PropertyVisibilityTests.cs" />
     <Compile Include="Serialization\ReadValueTests.cs" />
+    <Compile Include="Serialization\SampleTestData.OrderPayload.cs" />
     <Compile Include="Serialization\SpanTests.cs" />
     <Compile Include="Serialization\Stream.ReadTests.cs" />
     <Compile Include="Serialization\Stream.WriteTests.cs" />