From: Steve Harter Date: Thu, 25 Jul 2019 20:24:44 +0000 (-0500) Subject: Reduce memory usage of tests and fix asserts for escaped size (dotnet/corefx#39751) X-Git-Tag: submit/tizen/20210909.063632~11031^2~840 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=3cfddb1f5406e06e22e075526c343f7706b33ea7;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Reduce memory usage of tests and fix asserts for escaped size (dotnet/corefx#39751) Commit migrated from https://github.com/dotnet/corefx/commit/3e36efc9a3c3f22b87c5f33a25c0ad5dbfeb9396 --- diff --git a/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs b/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs index 85d345f..1fb5c62 100644 --- a/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs +++ b/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs @@ -231,6 +231,11 @@ namespace System.Buffers.Tests public static bool IsX64 { get; } = IntPtr.Size == 8; + // NOTE: InvalidAdvance_Large test is constrained to run on Windows and MacOSX because it causes + // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + // time the memory is accessed which triggers the full memory allocation. + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalFact(nameof(IsX64))] [OuterLoop] public void InvalidAdvance_Large() diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs index d295393..b6a019c 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs @@ -61,9 +61,10 @@ namespace System.Text.Json // All other UTF-16 characters can be represented by either 1 or 2 UTF-8 bytes. public const int MaxExpansionFactorWhileTranscoding = 3; - public const int MaxTokenSize = 1_000_000_000 / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes - public const int MaxBase46ValueTokenSize = (1_000_000_000 >> 2 * 3) / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes - public const int MaxCharacterTokenSize = 1_000_000_000 / MaxExpansionFactorWhileEscaping; // 166_666_666 characters + public const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. + public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes + public const int MaxBase46ValueTokenSize = (MaxEscapedTokenSize >> 2 * 3) / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes + public const int MaxCharacterTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 characters public const int MaximumFormatInt64Length = 20; // 19 + sign (i.e. -9223372036854775808) public const int MaximumFormatUInt64Length = 20; // i.e. 18446744073709551615 diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs index 74f3746..267cad6 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs @@ -66,20 +66,20 @@ namespace System.Text.Json public static void ThrowArgumentException(ReadOnlySpan propertyName, ReadOnlySpan value) { - if (propertyName.Length > JsonConstants.MaxTokenSize) + if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize) { ThrowArgumentException(SR.Format(SR.PropertyNameTooLarge, propertyName.Length)); } else { - Debug.Assert(value.Length > JsonConstants.MaxTokenSize); + Debug.Assert(value.Length > JsonConstants.MaxUnescapedTokenSize); ThrowArgumentException(SR.Format(SR.ValueTooLarge, value.Length)); } } public static void ThrowArgumentException(ReadOnlySpan propertyName, ReadOnlySpan value) { - if (propertyName.Length > JsonConstants.MaxTokenSize) + if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize) { ThrowArgumentException(SR.Format(SR.PropertyNameTooLarge, propertyName.Length)); } @@ -98,7 +98,7 @@ namespace System.Text.Json } else { - Debug.Assert(value.Length > JsonConstants.MaxTokenSize); + Debug.Assert(value.Length > JsonConstants.MaxUnescapedTokenSize); ThrowArgumentException(SR.Format(SR.ValueTooLarge, value.Length)); } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs index 41f844a..316175c 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs @@ -34,14 +34,14 @@ namespace System.Text.Json [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ValidateProperty(ReadOnlySpan propertyName) { - if (propertyName.Length > JsonConstants.MaxTokenSize) + if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize) ThrowHelper.ThrowArgumentException_PropertyNameTooLarge(propertyName.Length); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ValidateValue(ReadOnlySpan value) { - if (value.Length > JsonConstants.MaxTokenSize) + if (value.Length > JsonConstants.MaxUnescapedTokenSize) ThrowHelper.ThrowArgumentException_ValueTooLarge(value.Length); } @@ -95,21 +95,21 @@ namespace System.Text.Json [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ValidatePropertyAndValue(ReadOnlySpan propertyName, ReadOnlySpan value) { - if (propertyName.Length > JsonConstants.MaxCharacterTokenSize || value.Length > JsonConstants.MaxTokenSize) + if (propertyName.Length > JsonConstants.MaxCharacterTokenSize || value.Length > JsonConstants.MaxUnescapedTokenSize) ThrowHelper.ThrowArgumentException(propertyName, value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ValidatePropertyAndValue(ReadOnlySpan propertyName, ReadOnlySpan value) { - if (propertyName.Length > JsonConstants.MaxTokenSize || value.Length > JsonConstants.MaxCharacterTokenSize) + if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize || value.Length > JsonConstants.MaxCharacterTokenSize) ThrowHelper.ThrowArgumentException(propertyName, value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ValidatePropertyAndValue(ReadOnlySpan propertyName, ReadOnlySpan value) { - if (propertyName.Length > JsonConstants.MaxTokenSize || value.Length > JsonConstants.MaxTokenSize) + if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize || value.Length > JsonConstants.MaxUnescapedTokenSize) ThrowHelper.ThrowArgumentException(propertyName, value); } @@ -130,7 +130,7 @@ namespace System.Text.Json [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ValidatePropertyAndBytes(ReadOnlySpan propertyName, ReadOnlySpan bytes) { - if (propertyName.Length > JsonConstants.MaxTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize) + if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize) ThrowHelper.ThrowArgumentException(propertyName, bytes); } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs index d55076b..a54daea 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs @@ -23,7 +23,7 @@ namespace System.Text.Json private void WriteBase64StringHelper(ReadOnlySpan utf8PropertyName, ReadOnlySpan bytes) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); JsonWriterHelper.ValidateBytes(bytes); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTime.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTime.cs index 9e3db00..06c95a2 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTime.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTime.cs @@ -27,7 +27,7 @@ namespace System.Text.Json private void WriteStringHelper(ReadOnlySpan utf8PropertyName, DateTime value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteStringByOptions(utf8PropertyName, value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTimeOffset.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTimeOffset.cs index 7893df6..2894a69 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTimeOffset.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.DateTimeOffset.cs @@ -26,7 +26,7 @@ namespace System.Text.Json private void WriteStringHelper(ReadOnlySpan utf8PropertyName, DateTimeOffset value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteStringByOptions(utf8PropertyName, value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Decimal.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Decimal.cs index 8149596..d15cf69 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Decimal.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Decimal.cs @@ -26,7 +26,7 @@ namespace System.Text.Json private void WriteNumberHelper(ReadOnlySpan utf8PropertyName, decimal value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteNumberByOptions(utf8PropertyName, value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Double.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Double.cs index 2066a49..07f17fc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Double.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Double.cs @@ -26,7 +26,7 @@ namespace System.Text.Json private void WriteNumberHelper(ReadOnlySpan utf8PropertyName, double value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); JsonWriterHelper.ValidateDouble(value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Float.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Float.cs index 899fd9c..4882393 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Float.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Float.cs @@ -26,7 +26,7 @@ namespace System.Text.Json private void WriteNumberHelper(ReadOnlySpan utf8PropertyName, float value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); JsonWriterHelper.ValidateSingle(value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Guid.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Guid.cs index 386f546..6828db5 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Guid.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Guid.cs @@ -26,7 +26,7 @@ namespace System.Text.Json private void WriteStringHelper(ReadOnlySpan utf8PropertyName, Guid value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteStringByOptions(utf8PropertyName, value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs index 96418aa..81b8370 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs @@ -21,7 +21,7 @@ namespace System.Text.Json [MethodImpl(MethodImplOptions.AggressiveInlining)] private void ValidatePropertyNameAndDepth(ReadOnlySpan utf8PropertyName) { - if (utf8PropertyName.Length > JsonConstants.MaxTokenSize || CurrentDepth >= JsonConstants.MaxWriterDepth) + if (utf8PropertyName.Length > JsonConstants.MaxUnescapedTokenSize || CurrentDepth >= JsonConstants.MaxWriterDepth) ThrowHelper.ThrowInvalidOperationOrArgumentException(utf8PropertyName, _currentDepth); } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Literal.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Literal.cs index 63a3c45..d16539c 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Literal.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Literal.cs @@ -24,7 +24,7 @@ namespace System.Text.Json private void WriteLiteralHelper(ReadOnlySpan utf8PropertyName, ReadOnlySpan value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteLiteralByOptions(utf8PropertyName, value); @@ -300,7 +300,7 @@ namespace System.Text.Json private void WriteLiteralMinimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan value) { - Debug.Assert(value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(value.Length <= JsonConstants.MaxUnescapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileTranscoding) - value.Length - 4); // All ASCII, 2 quotes for property name, and 1 colon => escapedPropertyName.Length + value.Length + 3 @@ -331,7 +331,7 @@ namespace System.Text.Json private void WriteLiteralMinimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan value) { - Debug.Assert(value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(value.Length <= JsonConstants.MaxUnescapedTokenSize); Debug.Assert(escapedPropertyName.Length < int.MaxValue - value.Length - 4); int minRequired = escapedPropertyName.Length + value.Length + 3; // 2 quotes for property name, and 1 colon @@ -365,7 +365,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(value.Length <= JsonConstants.MaxUnescapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileTranscoding) - indent - value.Length - 5 - s_newLineLength); // All ASCII, 2 quotes for property name, 1 colon, and 1 space => escapedPropertyName.Length + value.Length + 4 @@ -411,7 +411,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(value.Length <= JsonConstants.MaxUnescapedTokenSize); Debug.Assert(escapedPropertyName.Length < int.MaxValue - indent - value.Length - 5 - s_newLineLength); int minRequired = indent + escapedPropertyName.Length + value.Length + 4; // 2 quotes for property name, 1 colon, and 1 space diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.SignedNumber.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.SignedNumber.cs index 04431e6..d74a1ec 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.SignedNumber.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.SignedNumber.cs @@ -26,7 +26,7 @@ namespace System.Text.Json private void WriteNumberHelper(ReadOnlySpan utf8PropertyName, long value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteNumberByOptions(utf8PropertyName, value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.String.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.String.cs index 950adb4..2dee64c 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.String.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.String.cs @@ -21,7 +21,7 @@ namespace System.Text.Json private void WritePropertyNameHelper(ReadOnlySpan utf8PropertyName) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteStringByOptionsPropertyName(utf8PropertyName); @@ -134,7 +134,7 @@ namespace System.Text.Json private void WriteStringMinimizedPropertyName(ReadOnlySpan escapedPropertyName) { - Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue - 4) / JsonConstants.MaxExpansionFactorWhileTranscoding); // All ASCII, 2 quotes for property name, and 1 colon => escapedPropertyName.Length + 3 @@ -165,7 +165,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue - 5 - indent - s_newLineLength) / JsonConstants.MaxExpansionFactorWhileTranscoding); // All ASCII, 2 quotes for property name, 1 colon, and 1 space => escapedPropertyName.Length + 4 @@ -287,7 +287,7 @@ namespace System.Text.Json private void WriteStringMinimizedPropertyName(ReadOnlySpan escapedPropertyName) { - Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < int.MaxValue - 4); int minRequired = escapedPropertyName.Length + 3; // 2 quotes for property name, and 1 colon @@ -318,7 +318,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedPropertyName.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < int.MaxValue - indent - 5 - s_newLineLength); int minRequired = indent + escapedPropertyName.Length + 4; // 2 quotes for property name, 1 colon, and 1 space @@ -369,7 +369,7 @@ namespace System.Text.Json private void WriteStringHelper(ReadOnlySpan utf8PropertyName, ReadOnlySpan utf8Value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize && utf8Value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize && utf8Value.Length <= JsonConstants.MaxUnescapedTokenSize); WriteStringByOptions(utf8PropertyName, utf8Value); @@ -536,7 +536,7 @@ namespace System.Text.Json private void WriteStringHelperEscapeValue(ReadOnlySpan utf8PropertyName, ReadOnlySpan value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); JsonWriterHelper.ValidateValue(value); @@ -620,7 +620,7 @@ namespace System.Text.Json private void WriteStringHelperEscapeValue(ReadOnlySpan utf8PropertyName, ReadOnlySpan utf8Value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); JsonWriterHelper.ValidateValue(utf8Value); @@ -704,7 +704,7 @@ namespace System.Text.Json private void WriteStringHelperEscapeProperty(ReadOnlySpan propertyName, ReadOnlySpan utf8Value) { - Debug.Assert(utf8Value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8Value.Length <= JsonConstants.MaxUnescapedTokenSize); JsonWriterHelper.ValidateProperty(propertyName); @@ -776,7 +776,7 @@ namespace System.Text.Json private void WriteStringHelperEscapeProperty(ReadOnlySpan utf8PropertyName, ReadOnlySpan utf8Value) { - Debug.Assert(utf8Value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8Value.Length <= JsonConstants.MaxUnescapedTokenSize); JsonWriterHelper.ValidateProperty(utf8PropertyName); @@ -1328,7 +1328,7 @@ namespace System.Text.Json // TODO: https://github.com/dotnet/corefx/issues/36958 private void WriteStringMinimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan escapedValue) { - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxUnescapedTokenSize); Debug.Assert(escapedPropertyName.Length < ((int.MaxValue - 6) / JsonConstants.MaxExpansionFactorWhileTranscoding) - escapedValue.Length); // All ASCII, 2 quotes for property name, 2 quotes for value, and 1 colon => escapedPropertyName.Length + escapedValue.Length + 5 @@ -1363,7 +1363,7 @@ namespace System.Text.Json // TODO: https://github.com/dotnet/corefx/issues/36958 private void WriteStringMinimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan escapedValue) { - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < int.MaxValue - escapedValue.Length - 6); int minRequired = escapedPropertyName.Length + escapedValue.Length + 5; // 2 quotes for property name, 2 quotes for value, and 1 colon @@ -1399,7 +1399,7 @@ namespace System.Text.Json // TODO: https://github.com/dotnet/corefx/issues/36958 private void WriteStringMinimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan escapedValue) { - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileTranscoding) - escapedValue.Length - 6); // All ASCII, 2 quotes for property name, 2 quotes for value, and 1 colon => escapedPropertyName.Length + escapedValue.Length + 5 @@ -1435,7 +1435,7 @@ namespace System.Text.Json // TODO: https://github.com/dotnet/corefx/issues/36958 private void WriteStringMinimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan escapedValue) { - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileTranscoding) - escapedValue.Length - 6); // All ASCII, 2 quotes for property name, 2 quotes for value, and 1 colon => escapedPropertyName.Length + escapedValue.Length + 5 @@ -1474,7 +1474,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < ((int.MaxValue - 7 - indent - s_newLineLength) / JsonConstants.MaxExpansionFactorWhileTranscoding) - escapedValue.Length); // All ASCII, 2 quotes for property name, 2 quotes for value, 1 colon, and 1 space => escapedPropertyName.Length + escapedValue.Length + 6 @@ -1524,7 +1524,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < int.MaxValue - indent - escapedValue.Length - 7 - s_newLineLength); int minRequired = indent + escapedPropertyName.Length + escapedValue.Length + 6; // 2 quotes for property name, 2 quotes for value, 1 colon, and 1 space @@ -1575,7 +1575,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileTranscoding) - escapedValue.Length - 7 - indent - s_newLineLength); // All ASCII, 2 quotes for property name, 2 quotes for value, 1 colon, and 1 space => escapedPropertyName.Length + escapedValue.Length + 6 @@ -1626,7 +1626,7 @@ namespace System.Text.Json int indent = Indentation; Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); - Debug.Assert(escapedValue.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(escapedValue.Length <= JsonConstants.MaxEscapedTokenSize); Debug.Assert(escapedPropertyName.Length < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileTranscoding) - escapedValue.Length - 7 - indent - s_newLineLength); // All ASCII, 2 quotes for property name, 2 quotes for value, 1 colon, and 1 space => escapedPropertyName.Length + escapedValue.Length + 6 diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.UnsignedNumber.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.UnsignedNumber.cs index dd02237..1acd0fc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.UnsignedNumber.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.UnsignedNumber.cs @@ -27,7 +27,7 @@ namespace System.Text.Json private void WriteNumberHelper(ReadOnlySpan utf8PropertyName, ulong value) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); WriteNumberByOptions(utf8PropertyName, value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.String.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.String.cs index dd8a04a..b23463a 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.String.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.String.cs @@ -21,7 +21,7 @@ namespace System.Text.Json private void WriteStringValueHelper(ReadOnlySpan utf8Value) { - Debug.Assert(utf8Value.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8Value.Length <= JsonConstants.MaxUnescapedTokenSize); WriteStringByOptions(utf8Value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs index 783d7a4..07df584 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs @@ -594,7 +594,7 @@ namespace System.Text.Json private void WriteStartHelper(ReadOnlySpan utf8PropertyName, byte token) { - Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxUnescapedTokenSize); ValidateDepth(); diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.ValueTextEquals.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.ValueTextEquals.cs index e84247b..d9ea93b 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.ValueTextEquals.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.ValueTextEquals.cs @@ -628,6 +628,11 @@ namespace System.Text.Json.Tests Assert.False(json.ValueTextEquals(lookup)); } + // NOTE: LookupOverflow test is constrained to run on Windows and MacOSX because it causes + // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + // time the memory is accessed which triggers the full memory allocation. + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalFact(nameof(IsX64))] [OuterLoop] public static void LookupOverflow() diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs index d6c50e6..eeeb70b 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs @@ -17,6 +17,10 @@ namespace System.Text.Json.Tests { public class Utf8JsonWriterTests { + private const int MaxExpansionFactorWhileEscaping = 6; + private const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. + private const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes + public static bool IsX64 { get; } = IntPtr.Size >= 8; [Theory] @@ -1605,6 +1609,11 @@ namespace System.Text.Json.Tests Assert.Throws(() => jsonUtf8.WriteStartArray(JsonEncodedText.Encode("name"))); } + // NOTE: WritingTooLargeProperty test is constrained to run on Windows and MacOSX because it causes + // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + // time the memory is accessed which triggers the full memory allocation. + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalTheory(nameof(IsX64))] [OuterLoop] [InlineData(true, true)] @@ -1618,8 +1627,8 @@ namespace System.Text.Json.Tests try { - key = new byte[1_000_000_000]; - keyChars = new char[1_000_000_000]; + key = new byte[MaxUnescapedTokenSize + 1]; + keyChars = new char[MaxUnescapedTokenSize + 1]; } catch (OutOfMemoryException) { @@ -1641,6 +1650,11 @@ namespace System.Text.Json.Tests Assert.Throws(() => jsonUtf8.WriteStartArray(key)); } + // NOTE: WritingTooLargePropertyStandalone test is constrained to run on Windows and MacOSX because it causes + // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + // time the memory is accessed which triggers the full memory allocation. + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalTheory(nameof(IsX64))] [OuterLoop] [InlineData(true, true)] @@ -1654,8 +1668,8 @@ namespace System.Text.Json.Tests try { - key = new byte[1_000_000_000]; - keyChars = new char[1_000_000_000]; + key = new byte[MaxUnescapedTokenSize + 1]; + keyChars = new char[MaxUnescapedTokenSize + 1]; } catch (OutOfMemoryException) { @@ -4358,6 +4372,11 @@ namespace System.Text.Json.Tests } } + // NOTE: WriteLargeKeyOrValue test is constrained to run on Windows and MacOSX because it causes + // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + // time the memory is accessed which triggers the full memory allocation. + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalTheory(nameof(IsX64))] [OuterLoop] [InlineData(true, true)] @@ -4371,8 +4390,8 @@ namespace System.Text.Json.Tests try { - key = new byte[1_000_000_001]; - value = new byte[1_000_000_001]; + key = new byte[MaxUnescapedTokenSize + 1]; + value = new byte[MaxUnescapedTokenSize + 1]; } catch (OutOfMemoryException) { @@ -4400,6 +4419,11 @@ namespace System.Text.Json.Tests } } + // NOTE: WriteLargeKeyValue test is constrained to run on Windows and MacOSX because it causes + // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + // time the memory is accessed which triggers the full memory allocation. + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalTheory(nameof(IsX64))] [OuterLoop] [InlineData(true, true)] @@ -4415,8 +4439,8 @@ namespace System.Text.Json.Tests try { - key = new byte[1_000_000_001]; - value = new byte[1_000_000_001]; + key = new byte[MaxUnescapedTokenSize + 1]; + value = new byte[MaxUnescapedTokenSize + 1]; } catch (OutOfMemoryException) { @@ -4424,11 +4448,43 @@ namespace System.Text.Json.Tests } WriteTooLargeHelper(options, key, value); - WriteTooLargeHelper(options, key.Slice(0, 1_000_000_000), value); - WriteTooLargeHelper(options, key, value.Slice(0, 1_000_000_000)); + WriteTooLargeHelper(options, key.Slice(0, MaxUnescapedTokenSize), value); + WriteTooLargeHelper(options, key, value.Slice(0, MaxUnescapedTokenSize)); WriteTooLargeHelper(options, key.Slice(0, 10_000_000 / 3), value.Slice(0, 10_000_000 / 3), noThrow: true); } + // NOTE: WriteLargeKeyEscapedValue test is constrained to run on Windows and MacOSX because it causes + // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + // time the memory is accessed which triggers the full memory allocation. + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalTheory(nameof(IsX64))] + [OuterLoop] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public void WriteLargeKeyEscapedValue(bool formatted, bool skipValidation) + { + var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation }; + + Span key; + Span value; + + try + { + // Since the byte values are 0 they will be escaped and size > MaxUnescapedTokenSize but < MaxEscapedTokenSize. + key = new byte[MaxUnescapedTokenSize / 2]; + value = new byte[MaxUnescapedTokenSize / 2]; + } + catch (OutOfMemoryException) + { + return; + } + + WriteTooLargeHelper(options, key, value, noThrow: true); + } + [Theory] [MemberData(nameof(JsonDateTimeTestData.DateTimeFractionTrimBaseTests), MemberType = typeof(JsonDateTimeTestData))] [MemberData(nameof(JsonDateTimeTestData.DateTimeFractionTrimUtcOffsetTests), MemberType = typeof(JsonDateTimeTestData))]