Fix the max token size threshold to correctly compute to 125MB for Base64 bytes....

* Fix the max token size threshold to correctly compute to 125MB for
Base64 bytes.

* Rename constant to fix transpose error: Base46 -> Base64

* Enable the outerloop tests for windows and osx only and update to use
platform specific new line.

Commit migrated from https://github.com/dotnet/corefx/commit/1511f723f7adfe82cef72460df45dc4422d303c6

diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs
index b6a019c..9a1805c 100644 (file)
--- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs
+++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs
@@ -63,7 +63,7 @@ namespace System.Text.Json
 
         public const int MaxEscapedTokenSize = 1_000_000_000;   // Max size for already escaped value.
         public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping;  // 166_666_666 bytes
-        public const int MaxBase46ValueTokenSize = (MaxEscapedTokenSize >> 2 * 3) / MaxExpansionFactorWhileEscaping;  // 125_000_000 bytes
+        public const int MaxBase64ValueTokenSize = (MaxEscapedTokenSize >> 2) * 3 / MaxExpansionFactorWhileEscaping;  // 125_000_000 bytes
         public const int MaxCharacterTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 characters
 
         public const int MaximumFormatInt64Length = 20;   // 19 + sign (i.e. -9223372036854775808)

diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs
index 316175c..d3be981 100644 (file)
--- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs
+++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs
@@ -48,7 +48,7 @@ namespace System.Text.Json
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void ValidateBytes(ReadOnlySpan<byte> bytes)
         {
-            if (bytes.Length > JsonConstants.MaxBase46ValueTokenSize)
+            if (bytes.Length > JsonConstants.MaxBase64ValueTokenSize)
                 ThrowHelper.ThrowArgumentException_ValueTooLarge(bytes.Length);
         }
 
@@ -123,14 +123,14 @@ namespace System.Text.Json
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void ValidatePropertyAndBytes(ReadOnlySpan<char> propertyName, ReadOnlySpan<byte> bytes)
         {
-            if (propertyName.Length > JsonConstants.MaxCharacterTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize)
+            if (propertyName.Length > JsonConstants.MaxCharacterTokenSize || bytes.Length > JsonConstants.MaxBase64ValueTokenSize)
                 ThrowHelper.ThrowArgumentException(propertyName, bytes);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void ValidatePropertyAndBytes(ReadOnlySpan<byte> propertyName, ReadOnlySpan<byte> bytes)
         {
-            if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize)
+            if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize || bytes.Length > JsonConstants.MaxBase64ValueTokenSize)
                 ThrowHelper.ThrowArgumentException(propertyName, bytes);
         }
 

diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs
index a5414b6..b79fac1 100644 (file)
--- a/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs
+++ b/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs
@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Buffers;
+using System.Buffers.Text;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
@@ -2466,6 +2467,11 @@ namespace System.Text.Json.Tests
 
             using (var jsonUtf8 = new Utf8JsonWriter(output, options))
             {
+                Assert.Throws<ArgumentException>(() => jsonUtf8.WriteBase64StringValue(value.AsSpan(0, 125_000_001)));
+            }
+
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
                 Assert.Throws<ArgumentException>(() => jsonUtf8.WriteBase64StringValue(value));
             }
 
@@ -2494,6 +2500,137 @@ namespace System.Text.Json.Tests
             }
         }
 
+        // NOTE: WritingTooLargeProperty test is constrained to run on Windows and MacOSX because it causes
+        //       problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can
+        //       succeed even if there is not enough memory but then the test may get killed by the OOM killer at the
+        //       time the memory is accessed which triggers the full memory allocation.
+        [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
+        [ConditionalTheory(nameof(IsX64))]
+        [OuterLoop]
+        [InlineData(true, true)]
+        [InlineData(true, false)]
+        [InlineData(false, true)]
+        [InlineData(false, false)]
+        public void WritingLargestPossibleBase64Bytes(bool formatted, bool skipValidation)
+        {
+            byte[] value;
+
+            try
+            {
+                value = new byte[125_000_000];
+            }
+            catch (OutOfMemoryException)
+            {
+                return;
+            }
+
+            value.AsSpan().Fill(168);
+
+            var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation };
+            var output = new ArrayBufferWriter<byte>(1024);
+
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteBase64StringValue(value);
+            }
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String("foo", value);
+                jsonUtf8.WriteEndObject();
+            }
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String(Encoding.UTF8.GetBytes("foo"), value);
+                jsonUtf8.WriteEndObject();
+            }
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String("foo".AsSpan(), value);
+                jsonUtf8.WriteEndObject();
+            }
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String(JsonEncodedText.Encode("foo"), value);
+                jsonUtf8.WriteEndObject();
+            }
+        }
+
+        // https://github.com/dotnet/corefx/issues/40755
+        [Theory]
+        [InlineData(true, true)]
+        [InlineData(true, false)]
+        [InlineData(false, true)]
+        [InlineData(false, false)]
+        public void Writing3MBBase64Bytes(bool formatted, bool skipValidation)
+        {
+            byte[] value = new byte[3 * 1024 * 1024];
+
+            value.AsSpan().Fill(168);
+
+            byte[] base64StringUtf8 = new byte[Base64.GetMaxEncodedToUtf8Length(value.Length)];
+            Base64.EncodeToUtf8(value, base64StringUtf8, out _, out int bytesWritten);
+            string expectedValue = Encoding.UTF8.GetString(base64StringUtf8.AsSpan(0, bytesWritten).ToArray());
+
+            string expectedJson = formatted ? $"{{{Environment.NewLine}  \"foo\": \"{expectedValue}\"{Environment.NewLine}}}" : $"{{\"foo\":\"{expectedValue}\"}}";
+
+            var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation };
+            var output = new ArrayBufferWriter<byte>(1024);
+
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteBase64StringValue(value);
+            }
+            JsonTestHelper.AssertContents($"\"{expectedValue}\"", output);
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String("foo", value);
+                jsonUtf8.WriteEndObject();
+            }
+            JsonTestHelper.AssertContents(expectedJson, output);
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String(Encoding.UTF8.GetBytes("foo"), value);
+                jsonUtf8.WriteEndObject();
+            }
+            JsonTestHelper.AssertContents(expectedJson, output);
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String("foo".AsSpan(), value);
+                jsonUtf8.WriteEndObject();
+            }
+            JsonTestHelper.AssertContents(expectedJson, output);
+
+            output.Clear();
+            using (var jsonUtf8 = new Utf8JsonWriter(output, options))
+            {
+                jsonUtf8.WriteStartObject();
+                jsonUtf8.WriteBase64String(JsonEncodedText.Encode("foo"), value);
+                jsonUtf8.WriteEndObject();
+            }
+            JsonTestHelper.AssertContents(expectedJson, output);
+        }
+
         [Theory]
         [InlineData(true, true)]
         [InlineData(true, false)]
@@ -3932,7 +4069,7 @@ namespace System.Text.Json.Tests
             JsonTestHelper.AssertContents("{" + ValidUtf8Expected + ":" + ValidUtf8Expected + "}", output);
         }
 
-        private static readonly string s_InvalidUtf16Input = new string (new char[2] { (char)0xD801, 'a' });
+        private static readonly string s_InvalidUtf16Input = new string(new char[2] { (char)0xD801, 'a' });
         private const string InvalidUtf16Expected = "\"\\uFFFDa\"";
 
         private static readonly string s_ValidUtf16Input = new string(new char[2] { (char)0xD801, (char)0xDC37 }); // 0x10437