From bb18f7869f7dd9fa84cf9930bac4629a68bacf7c Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Thu, 30 May 2019 22:13:31 -0700 Subject: [PATCH] Add Base64 APIs to Utf8JsonReader, Utf8JsonWriter, and JsonElement (dotnet/corefx#38048) * Add Utf8JsonWriter Base64 APIs. * Add Utf8JsonReader Base64 APIs. * Add JsonElement Base64 APIs. * Update API shape based on review. * Auto-generate the ref assembly. * Address PR feedback so far. * Add escaping step and update length counters accordingly. * Add JsonWriter API tests. * Add JsonReader and JsonElement tests. Commit migrated from https://github.com/dotnet/corefx/commit/82408cd90f4d4573d502e8df2ca437b35e6a37f7 --- .../System.Text.Json/ref/System.Text.Json.cs | 15 +- .../System.Text.Json/src/Resources/Strings.resx | 3 + .../System.Text.Json/src/System.Text.Json.csproj | 4 +- .../src/System/Text/Json/Document/JsonDocument.cs | 23 ++ .../src/System/Text/Json/Document/JsonElement.cs | 52 +++ .../src/System/Text/Json/JsonConstants.cs | 5 +- .../Json/Reader/JsonReaderHelper.Unescaping.cs | 72 ++++ .../Text/Json/Reader/Utf8JsonReader.TryGet.cs | 89 +++-- .../src/System/Text/Json/ThrowHelper.cs | 6 +- .../Text/Json/Writer/JsonWriterHelper.Escaping.cs | 2 +- .../System/Text/Json/Writer/JsonWriterHelper.cs | 21 ++ .../Writer/Utf8JsonWriter.WriteProperties.Bytes.cs | 376 +++++++++++++++++++++ .../Writer/Utf8JsonWriter.WriteValues.Bytes.cs | 119 +++++++ .../Writer/Utf8JsonWriter.WriteValues.Helpers.cs | 39 +++ .../System.Text.Json/tests/JsonDocumentTests.cs | 162 +++++++-- .../tests/Utf8JsonReaderTests.TryGet.cs | 180 ++++++++++ .../System.Text.Json/tests/Utf8JsonWriterTests.cs | 339 ++++++++++++++++++- 17 files changed, 1455 insertions(+), 52 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Bytes.cs diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index fbf9ea8..664f72a 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -38,6 +38,7 @@ namespace System.Text.Json public System.Text.Json.JsonElement.ObjectEnumerator EnumerateObject() { throw null; } public int GetArrayLength() { throw null; } public bool GetBoolean() { throw null; } + public byte[] GetBytesFromBase64() { throw null; } public System.DateTime GetDateTime() { throw null; } public System.DateTimeOffset GetDateTimeOffset() { throw null; } public decimal GetDecimal() { throw null; } @@ -56,6 +57,7 @@ namespace System.Text.Json [System.CLSCompliantAttribute(false)] public ulong GetUInt64() { throw null; } public override string ToString() { throw null; } + public bool TryGetBytesFromBase64(out byte[] value) { throw null; } public bool TryGetDateTime(out System.DateTime value) { throw null; } public bool TryGetDateTimeOffset(out System.DateTimeOffset value) { throw null; } public bool TryGetDecimal(out decimal value) { throw null; } @@ -196,9 +198,10 @@ namespace System.Text.Json public System.Buffers.ReadOnlySequence ValueSequence { get { throw null; } } public System.ReadOnlySpan ValueSpan { get { throw null; } } public bool GetBoolean() { throw null; } + public byte[] GetBytesFromBase64() { throw null; } + public string GetComment() { throw null; } public System.DateTime GetDateTime() { throw null; } public System.DateTimeOffset GetDateTimeOffset() { throw null; } - public string GetComment() { throw null; } public decimal GetDecimal() { throw null; } public double GetDouble() { throw null; } public System.Guid GetGuid() { throw null; } @@ -214,6 +217,7 @@ namespace System.Text.Json public void Skip() { } public bool TextEquals(System.ReadOnlySpan otherUtf8Text) { throw null; } public bool TextEquals(System.ReadOnlySpan otherText) { throw null; } + public bool TryGetBytesFromBase64(out byte[] value) { throw null; } public bool TryGetDateTime(out System.DateTime value) { throw null; } public bool TryGetDateTimeOffset(out System.DateTimeOffset value) { throw null; } public bool TryGetDecimal(out decimal value) { throw null; } @@ -242,6 +246,11 @@ namespace System.Text.Json public void Reset() { } public void Reset(System.Buffers.IBufferWriter bufferWriter) { } public void Reset(System.IO.Stream utf8Json) { } + public void WriteBase64String(System.ReadOnlySpan utf8PropertyName, System.ReadOnlySpan bytes) { } + public void WriteBase64String(System.ReadOnlySpan propertyName, System.ReadOnlySpan bytes) { } + public void WriteBase64String(string propertyName, System.ReadOnlySpan bytes) { } + public void WriteBase64String(System.Text.Json.JsonEncodedText propertyName, System.ReadOnlySpan bytes) { } + public void WriteBase64StringValue(System.ReadOnlySpan bytes) { } public void WriteBoolean(System.ReadOnlySpan utf8PropertyName, bool value) { } public void WriteBoolean(System.ReadOnlySpan propertyName, bool value) { } public void WriteBoolean(string propertyName, bool value) { } @@ -385,10 +394,10 @@ namespace System.Text.Json.Serialization public static TValue Parse(string json, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; } public static System.Threading.Tasks.ValueTask ReadAsync(System.IO.Stream utf8Json, System.Type returnType, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.ValueTask ReadAsync(System.IO.Stream utf8Json, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public static byte[] ToUtf8Bytes(object value, System.Type type, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; } - public static byte[] ToUtf8Bytes(TValue value, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; } public static string ToString(object value, System.Type type, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; } public static string ToString(TValue value, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; } + public static byte[] ToUtf8Bytes(object value, System.Type type, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; } + public static byte[] ToUtf8Bytes(TValue value, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; } public static System.Threading.Tasks.Task WriteAsync(object value, System.Type type, System.IO.Stream utf8Json, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.Task WriteAsync(TValue value, System.IO.Stream utf8Json, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } } diff --git a/src/libraries/System.Text.Json/src/Resources/Strings.resx b/src/libraries/System.Text.Json/src/Resources/Strings.resx index f0d121a..6f8b9bc 100644 --- a/src/libraries/System.Text.Json/src/Resources/Strings.resx +++ b/src/libraries/System.Text.Json/src/Resources/Strings.resx @@ -137,6 +137,9 @@ Cannot transcode invalid UTF-8 JSON text to UTF-16 string. + + Cannot decode JSON text that is not encoded as valid Base64 to bytes. + Cannot transcode invalid UTF-16 string to UTF-8 JSON text. diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 5bfb8e6..941b0ba 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -11,7 +11,7 @@ - + @@ -124,6 +124,7 @@ + @@ -136,6 +137,7 @@ + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs index 1d4e447..b5ffa48 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs @@ -263,6 +263,29 @@ namespace System.Text.Json return GetString(index - DbRow.Size, JsonTokenType.PropertyName); } + internal bool TryGetValue(int index, out byte[] value) + { + CheckNotDisposed(); + + DbRow row = _parsedData.Get(index); + + CheckExpectedType(JsonTokenType.String, row.TokenType); + + ReadOnlySpan data = _utf8Json.Span; + ReadOnlySpan segment = data.Slice(row.Location, row.SizeOrLength); + + // Segment needs to be unescaped + if (row.HasComplexChildren) + { + int idx = segment.IndexOf(JsonConstants.BackSlash); + Debug.Assert(idx != -1); + return JsonReaderHelper.TryGetUnescapedBase64Bytes(segment, idx, out value); + } + + Debug.Assert(segment.IndexOf(JsonConstants.BackSlash) == -1); + return JsonReaderHelper.TryDecodeBase64(segment, out value); + } + internal bool TryGetValue(int index, out int value) { CheckNotDisposed(); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs index 34be938..e9d12eb 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Buffers.Text; using System.Collections.Generic; using System.Diagnostics; @@ -351,6 +352,57 @@ namespace System.Text.Json } /// + /// Attempts to represent the current JSON string as bytes assuming it is base 64 encoded. + /// + /// Receives the value. + /// + /// This method does not create a byte[] representation of values other than bsae 64 encoded JSON strings. + /// + /// + /// if the entire token value is encoded as valid base 64 text and can be successfully decoded to bytes. + /// otherwise. + /// + /// + /// This value's is not . + /// + /// + /// The parent has been disposed. + /// + public bool TryGetBytesFromBase64(out byte[] value) + { + CheckValidInstance(); + + return _parent.TryGetValue(_idx, out value); + } + + /// + /// Gets the value of the element as bytes. + /// + /// + /// This method does not create a byte[] representation of values other than base 64 encoded JSON strings. + /// + /// The value decode to bytes. + /// + /// This value's is not . + /// + /// + /// The value is not encoded as base 64 text and hence cannot be decoded to bytes. + /// + /// + /// The parent has been disposed. + /// + /// + public byte[] GetBytesFromBase64() + { + if (TryGetBytesFromBase64(out byte[] value)) + { + return value; + } + + throw new FormatException(); + } + + /// /// Attempts to represent the current JSON number as an . /// /// Receives the value. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs index ec65999..bede535 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs @@ -57,8 +57,9 @@ namespace System.Text.Json // All other UTF-16 characters can be represented by either 1 or 2 UTF-8 bytes. public const int MaxExpansionFactorWhileTranscoding = 3; - public const int MaxTokenSize = 2_000_000_000 / MaxExpansionFactorWhileEscaping; // 357_913_941 bytes - public const int MaxCharacterTokenSize = 2_000_000_000 / MaxExpansionFactorWhileEscaping; // 357_913_941 characters + public const int MaxTokenSize = 1_000_000_000 / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes + public const int MaxBase46ValueTokenSize = (1_000_000_000 >> 2 * 3) / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes + public const int MaxCharacterTokenSize = 1_000_000_000 / MaxExpansionFactorWhileEscaping; // 166_666_666 characters public const int MaximumFormatInt64Length = 20; // 19 + sign (i.e. -9223372036854775808) public const int MaximumFormatUInt64Length = 20; // i.e. 18446744073709551615 diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderHelper.Unescaping.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderHelper.Unescaping.cs index e7025e7..fa4502b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderHelper.Unescaping.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderHelper.Unescaping.cs @@ -10,6 +10,31 @@ namespace System.Text.Json { internal static partial class JsonReaderHelper { + public static bool TryGetUnescapedBase64Bytes(ReadOnlySpan utf8Source, int idx, out byte[] bytes) + { + byte[] unescapedArray = null; + + Span utf8Unescaped = utf8Source.Length <= JsonConstants.StackallocThreshold ? + stackalloc byte[utf8Source.Length] : + (unescapedArray = ArrayPool.Shared.Rent(utf8Source.Length)); + + Unescape(utf8Source, utf8Unescaped, idx, out int written); + Debug.Assert(written > 0); + + utf8Unescaped = utf8Unescaped.Slice(0, written); + Debug.Assert(!utf8Unescaped.IsEmpty); + + bool result = TryDecodeBase64InPlace(utf8Unescaped, out bytes); + + if (unescapedArray != null) + { + utf8Unescaped.Clear(); + ArrayPool.Shared.Return(unescapedArray); + } + + return result; + } + // Reject any invalid UTF-8 data rather than silently replacing. public static readonly UTF8Encoding s_utf8Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); @@ -107,6 +132,53 @@ namespace System.Text.Json return result; } + public static bool TryDecodeBase64InPlace(Span utf8Unescaped, out byte[] bytes) + { + OperationStatus status = Base64.DecodeFromUtf8InPlace(utf8Unescaped, out int bytesWritten); + if (status != OperationStatus.Done) + { + bytes = null; + return false; + } + bytes = utf8Unescaped.Slice(0, bytesWritten).ToArray(); + return true; + } + + public static bool TryDecodeBase64(ReadOnlySpan utf8Unescaped, out byte[] bytes) + { + byte[] pooledArray = null; + + Span byteSpan = utf8Unescaped.Length <= JsonConstants.StackallocThreshold ? + stackalloc byte[utf8Unescaped.Length] : + (pooledArray = ArrayPool.Shared.Rent(utf8Unescaped.Length)); + + OperationStatus status = Base64.DecodeFromUtf8(utf8Unescaped, byteSpan, out int bytesConsumed, out int bytesWritten); + + if (status != OperationStatus.Done) + { + bytes = null; + + if (pooledArray != null) + { + byteSpan.Clear(); + ArrayPool.Shared.Return(pooledArray); + } + + return false; + } + Debug.Assert(bytesConsumed == utf8Unescaped.Length); + + bytes = byteSpan.Slice(0, bytesWritten).ToArray(); + + if (pooledArray != null) + { + byteSpan.Clear(); + ArrayPool.Shared.Return(pooledArray); + } + + return true; + } + public static string TranscodeHelper(ReadOnlySpan utf8Unescaped) { try diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs index d4e25a7..2151d40 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs @@ -58,7 +58,7 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the TokenType is JsonTokenType.True and false if the TokenType is JsonTokenType.False. + /// Returns if the TokenType is JsonTokenType.True and if the TokenType is JsonTokenType.False. /// /// /// Thrown if trying to get the value of a JSON token that is not a boolean (i.e. or ). @@ -85,6 +85,27 @@ namespace System.Text.Json } /// + /// Parses the current JSON token value from the source and decodes the base 64 encoded JSON string as bytes. + /// + /// + /// Thrown if trying to get the value of a JSON token that is not a . + /// + /// + /// + /// Thrown when the JSON string contains data outside of the expected base 64 range, or if it contains invalid/more than two padding characters, + /// or is incomplete (i.e. the JSON string length is not a multiple of 4). + /// + public byte[] GetBytesFromBase64() + { + if (!TryGetBytesFromBase64(out byte[] value)) + { + throw ThrowHelper.GetFormatException(DateType.Base64String); + } + + return value; + } + + /// /// Parses the current JSON token value from the source as an . /// Returns the value if the entire UTF-8 encoded token value can be successfully parsed to an /// value. @@ -321,10 +342,40 @@ namespace System.Text.Json } /// + /// Parses the current JSON token value from the source and decodes the base 64 encoded JSON string as bytes. + /// Returns if the entire token value is encoded as valid base 64 text and can be successfully + /// decoded to bytes. + /// Returns otherwise. + /// + /// + /// Thrown if trying to get the value of a JSON token that is not a . + /// + /// + public bool TryGetBytesFromBase64(out byte[] value) + { + if (TokenType != JsonTokenType.String) + { + throw ThrowHelper.GetInvalidOperationException_ExpectedString(TokenType); + } + + ReadOnlySpan span = HasValueSequence ? ValueSequence.ToArray() : ValueSpan; + + if (_stringHasEscaping) + { + int idx = span.IndexOf(JsonConstants.BackSlash); + Debug.Assert(idx != -1); + return JsonReaderHelper.TryGetUnescapedBase64Bytes(span, idx, out value); + } + + Debug.Assert(span.IndexOf(JsonConstants.BackSlash) == -1); + return JsonReaderHelper.TryDecodeBase64(span, out value); + } + + /// /// Parses the current JSON token value from the source as an . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to an value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -343,9 +394,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -364,9 +415,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -386,9 +437,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -408,9 +459,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -429,9 +480,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -450,9 +501,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -471,9 +522,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . @@ -530,9 +581,9 @@ namespace System.Text.Json /// /// Parses the current JSON token value from the source as a . - /// Returns true if the entire UTF-8 encoded token value can be successfully + /// Returns if the entire UTF-8 encoded token value can be successfully /// parsed to a value. - /// Returns false otherwise. + /// Returns otherwise. /// /// /// Thrown if trying to get the value of a JSON token that is not a . diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs index afc8ba0..efed281f3 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs @@ -517,6 +517,9 @@ namespace System.Text.Json case DateType.DateTimeOffset: message = SR.FormatDateTimeOffset; break; + case DateType.Base64String: + message = SR.CannotDecodeInvalidBase64; + break; default: Debug.Fail($"The DateType enum value: {dateType} is not part of the switch. Add the appropriate case and exception message."); break; @@ -580,6 +583,7 @@ namespace System.Text.Json internal enum DateType { DateTime, - DateTimeOffset + DateTimeOffset, + Base64String } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.Escaping.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.Escaping.cs index 891bd7e..fceb5f0 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.Escaping.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.Escaping.cs @@ -20,7 +20,7 @@ namespace System.Text.Json private static ReadOnlySpan AllowList => new byte[byte.MaxValue + 1] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, + 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs index 82b67a4..b9e64ca 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs @@ -46,6 +46,13 @@ namespace System.Text.Json } [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ValidateBytes(ReadOnlySpan bytes) + { + if (bytes.Length > JsonConstants.MaxBase46ValueTokenSize) + ThrowHelper.ThrowArgumentException_ValueTooLarge(bytes.Length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ValidateDouble(double value) { #if BUILDING_INBOX_LIBRARY @@ -113,6 +120,20 @@ namespace System.Text.Json ThrowHelper.ThrowArgumentException(propertyName, value); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ValidatePropertyAndBytes(ReadOnlySpan propertyName, ReadOnlySpan bytes) + { + if (propertyName.Length > JsonConstants.MaxCharacterTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize) + ThrowHelper.ThrowArgumentException(propertyName, bytes); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ValidatePropertyAndBytes(ReadOnlySpan propertyName, ReadOnlySpan bytes) + { + if (propertyName.Length > JsonConstants.MaxTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize) + ThrowHelper.ThrowArgumentException(propertyName, bytes); + } + internal static void ValidateNumber(ReadOnlySpan utf8FormattedNumber) { // This is a simplified version of the number reader from Utf8JsonReader.TryGetNumber, diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs new file mode 100644 index 0000000..dd4cf73 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Bytes.cs @@ -0,0 +1,376 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Buffers.Text; +using System.Diagnostics; + +namespace System.Text.Json +{ + public sealed partial class Utf8JsonWriter + { + /// + /// Writes the pre-encoded property name and raw bytes value (as a base 64 encoded JSON string) as part of a name/value pair of a JSON object. + /// + /// The JSON encoded property name of the JSON object to be transcoded and written as UTF-8. + /// The binary data to be written as a base 64 encoded JSON string as part of the name/value pair. + /// + /// The property name should already be escaped when the instance of was created. + /// + /// + /// Thrown if this would result in an invalid JSON to be written (while validation is enabled). + /// + public void WriteBase64String(JsonEncodedText propertyName, ReadOnlySpan bytes) + => WriteBase64StringHelper(propertyName.EncodedUtf8Bytes, bytes); + + private void WriteBase64StringHelper(ReadOnlySpan utf8PropertyName, ReadOnlySpan bytes) + { + Debug.Assert(utf8PropertyName.Length <= JsonConstants.MaxTokenSize); + + JsonWriterHelper.ValidateBytes(bytes); + + WriteBase64ByOptions(utf8PropertyName, bytes); + + SetFlagToAddListSeparatorBeforeNextItem(); + _tokenType = JsonTokenType.String; + } + + /// + /// Writes the property name and raw bytes value (as a Base64 encoded JSON string) as part of a name/value pair of a JSON object. + /// + /// The property name of the JSON object to be transcoded and written as UTF-8. + /// The binary data to be written as a base 64 encoded JSON string as part of the name/value pair. + /// + /// The property name is escaped before writing. + /// + /// + /// Thrown when the specified property name is too large. + /// + /// + /// Thrown if this would result in an invalid JSON to be written (while validation is enabled). + /// + public void WriteBase64String(string propertyName, ReadOnlySpan bytes) + => WriteBase64String(propertyName.AsSpan(), bytes); + + /// + /// Writes the property name and raw bytes value (as a base 64 encoded JSON string) as part of a name/value pair of a JSON object. + /// + /// The property name of the JSON object to be transcoded and written as UTF-8. + /// The binary data to be written as a base 64 encoded JSON string as part of the name/value pair. + /// + /// The property name is escaped before writing. + /// + /// + /// Thrown when the specified property name is too large. + /// + /// + /// Thrown if this would result in an invalid JSON to be written (while validation is enabled). + /// + public void WriteBase64String(ReadOnlySpan propertyName, ReadOnlySpan bytes) + { + JsonWriterHelper.ValidatePropertyAndBytes(propertyName, bytes); + + WriteBase64Escape(propertyName, bytes); + + SetFlagToAddListSeparatorBeforeNextItem(); + _tokenType = JsonTokenType.String; + } + + /// + /// Writes the property name and raw bytes value (as a base 64 encoded JSON string) as part of a name/value pair of a JSON object. + /// + /// The UTF-8 encoded property name of the JSON object to be written. + /// The binary data to be written as a base 64 encoded JSON string as part of the name/value pair. + /// + /// The property name is escaped before writing. + /// + /// + /// Thrown when the specified property name is too large. + /// + /// + /// Thrown if this would result in an invalid JSON to be written (while validation is enabled). + /// + public void WriteBase64String(ReadOnlySpan utf8PropertyName, ReadOnlySpan bytes) + { + JsonWriterHelper.ValidatePropertyAndBytes(utf8PropertyName, bytes); + + WriteBase64Escape(utf8PropertyName, bytes); + + SetFlagToAddListSeparatorBeforeNextItem(); + _tokenType = JsonTokenType.String; + } + + private void WriteBase64Escape(ReadOnlySpan propertyName, ReadOnlySpan bytes) + { + int propertyIdx = JsonWriterHelper.NeedsEscaping(propertyName); + + Debug.Assert(propertyIdx >= -1 && propertyIdx < propertyName.Length); + + if (propertyIdx != -1) + { + WriteBase64EscapeProperty(propertyName, bytes, propertyIdx); + } + else + { + WriteBase64ByOptions(propertyName, bytes); + } + } + + private void WriteBase64Escape(ReadOnlySpan utf8PropertyName, ReadOnlySpan bytes) + { + int propertyIdx = JsonWriterHelper.NeedsEscaping(utf8PropertyName); + + Debug.Assert(propertyIdx >= -1 && propertyIdx < utf8PropertyName.Length); + + if (propertyIdx != -1) + { + WriteBase64EscapeProperty(utf8PropertyName, bytes, propertyIdx); + } + else + { + WriteBase64ByOptions(utf8PropertyName, bytes); + } + } + + private void WriteBase64EscapeProperty(ReadOnlySpan propertyName, ReadOnlySpan bytes, int firstEscapeIndexProp) + { + Debug.Assert(int.MaxValue / JsonConstants.MaxExpansionFactorWhileEscaping >= propertyName.Length); + Debug.Assert(firstEscapeIndexProp >= 0 && firstEscapeIndexProp < propertyName.Length); + + char[] propertyArray = null; + + int length = JsonWriterHelper.GetMaxEscapedLength(propertyName.Length, firstEscapeIndexProp); + + Span escapedPropertyName = length <= JsonConstants.StackallocThreshold ? + stackalloc char[length] : + (propertyArray = ArrayPool.Shared.Rent(length)); + + JsonWriterHelper.EscapeString(propertyName, escapedPropertyName, firstEscapeIndexProp, out int written); + + WriteBase64ByOptions(escapedPropertyName.Slice(0, written), bytes); + + if (propertyArray != null) + { + ArrayPool.Shared.Return(propertyArray); + } + } + + private void WriteBase64EscapeProperty(ReadOnlySpan utf8PropertyName, ReadOnlySpan bytes, int firstEscapeIndexProp) + { + Debug.Assert(int.MaxValue / JsonConstants.MaxExpansionFactorWhileEscaping >= utf8PropertyName.Length); + Debug.Assert(firstEscapeIndexProp >= 0 && firstEscapeIndexProp < utf8PropertyName.Length); + + byte[] propertyArray = null; + + int length = JsonWriterHelper.GetMaxEscapedLength(utf8PropertyName.Length, firstEscapeIndexProp); + + Span escapedPropertyName = length <= JsonConstants.StackallocThreshold ? + stackalloc byte[length] : + (propertyArray = ArrayPool.Shared.Rent(length)); + + JsonWriterHelper.EscapeString(utf8PropertyName, escapedPropertyName, firstEscapeIndexProp, out int written); + + WriteBase64ByOptions(escapedPropertyName.Slice(0, written), bytes); + + if (propertyArray != null) + { + ArrayPool.Shared.Return(propertyArray); + } + } + + private void WriteBase64ByOptions(ReadOnlySpan propertyName, ReadOnlySpan bytes) + { + ValidateWritingProperty(); + if (Options.Indented) + { + WriteBase64Indented(propertyName, bytes); + } + else + { + WriteBase64Minimized(propertyName, bytes); + } + } + + private void WriteBase64ByOptions(ReadOnlySpan utf8PropertyName, ReadOnlySpan bytes) + { + ValidateWritingProperty(); + if (Options.Indented) + { + WriteBase64Indented(utf8PropertyName, bytes); + } + else + { + WriteBase64Minimized(utf8PropertyName, bytes); + } + } + + private void WriteBase64Minimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan bytes) + { + int encodedLength = Base64.GetMaxEncodedToUtf8Length(bytes.Length); + + Debug.Assert(escapedPropertyName.Length * JsonConstants.MaxExpansionFactorWhileTranscoding < int.MaxValue - (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) - 6); + + // All ASCII, 2 quotes for property name, 2 quotes to surround the base-64 encoded string value, and 1 colon => escapedPropertyName.Length + encodedLength + 5 + // Optionally, 1 list separator, and up to 3x growth when transcoding, with escaping which can by up to 6x. + int maxRequired = (escapedPropertyName.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) + (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) + 6; + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + output[BytesPending++] = JsonConstants.Quote; + + TranscodeAndWrite(escapedPropertyName, output); + + output[BytesPending++] = JsonConstants.Quote; + output[BytesPending++] = JsonConstants.KeyValueSeperator; + + output[BytesPending++] = JsonConstants.Quote; + + Base64EncodeAndWrite(bytes, output, encodedLength); + + output[BytesPending++] = JsonConstants.Quote; + } + + private void WriteBase64Minimized(ReadOnlySpan escapedPropertyName, ReadOnlySpan bytes) + { + int encodedLength = Base64.GetMaxEncodedToUtf8Length(bytes.Length); + + Debug.Assert(escapedPropertyName.Length < int.MaxValue - (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) - 6); + + // 2 quotes for property name, 2 quotes to surround the base-64 encoded string value, and 1 colon => escapedPropertyName.Length + encodedLength + 5 + // Optionally, 1 list separator, with escaping which can by up to 6x. + int maxRequired = escapedPropertyName.Length + (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) + 6; + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + output[BytesPending++] = JsonConstants.Quote; + + escapedPropertyName.CopyTo(output.Slice(BytesPending)); + BytesPending += escapedPropertyName.Length; + + output[BytesPending++] = JsonConstants.Quote; + output[BytesPending++] = JsonConstants.KeyValueSeperator; + + output[BytesPending++] = JsonConstants.Quote; + + Base64EncodeAndWrite(bytes, output, encodedLength); + + output[BytesPending++] = JsonConstants.Quote; + } + + private void WriteBase64Indented(ReadOnlySpan escapedPropertyName, ReadOnlySpan bytes) + { + int indent = Indentation; + Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); + + int encodedLength = Base64.GetMaxEncodedToUtf8Length(bytes.Length); + + Debug.Assert(escapedPropertyName.Length * JsonConstants.MaxExpansionFactorWhileTranscoding < int.MaxValue - indent - (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) - 7 - s_newLineLength); + + // All ASCII, 2 quotes for property name, 2 quotes to surround the base-64 encoded string value, 1 colon, and 1 space => indent + escapedPropertyName.Length + encodedLength + 6 + // Optionally, 1 list separator, 1-2 bytes for new line, and up to 3x growth when transcoding, with escaping which can by up to 6x. + int maxRequired = indent + (escapedPropertyName.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) + (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) + 7 + s_newLineLength; + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + + if (_tokenType != JsonTokenType.None) + { + WriteNewLine(output); + } + + JsonWriterHelper.WriteIndentation(output.Slice(BytesPending), indent); + BytesPending += indent; + + output[BytesPending++] = JsonConstants.Quote; + + TranscodeAndWrite(escapedPropertyName, output); + + output[BytesPending++] = JsonConstants.Quote; + output[BytesPending++] = JsonConstants.KeyValueSeperator; + output[BytesPending++] = JsonConstants.Space; + + output[BytesPending++] = JsonConstants.Quote; + + Base64EncodeAndWrite(bytes, output, encodedLength); + + output[BytesPending++] = JsonConstants.Quote; + } + + private void WriteBase64Indented(ReadOnlySpan escapedPropertyName, ReadOnlySpan bytes) + { + int indent = Indentation; + Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); + + int encodedLength = Base64.GetMaxEncodedToUtf8Length(bytes.Length); + + Debug.Assert(escapedPropertyName.Length < int.MaxValue - indent - (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) - 7 - s_newLineLength); + + // 2 quotes for property name, 2 quotes to surround the base-64 encoded string value, 1 colon, and 1 space => indent + escapedPropertyName.Length + encodedLength + 6 + // Optionally, 1 list separator, and 1-2 bytes for new line, with escaping which can by up to 6x. + int maxRequired = indent + escapedPropertyName.Length + (encodedLength * JsonConstants.MaxExpansionFactorWhileEscaping) + 7 + s_newLineLength; + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + + if (_tokenType != JsonTokenType.None) + { + WriteNewLine(output); + } + + JsonWriterHelper.WriteIndentation(output.Slice(BytesPending), indent); + BytesPending += indent; + + output[BytesPending++] = JsonConstants.Quote; + + escapedPropertyName.CopyTo(output.Slice(BytesPending)); + BytesPending += escapedPropertyName.Length; + + output[BytesPending++] = JsonConstants.Quote; + output[BytesPending++] = JsonConstants.KeyValueSeperator; + output[BytesPending++] = JsonConstants.Space; + + output[BytesPending++] = JsonConstants.Quote; + + Base64EncodeAndWrite(bytes, output, encodedLength); + + output[BytesPending++] = JsonConstants.Quote; + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Bytes.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Bytes.cs new file mode 100644 index 0000000..ef1fdb8 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Bytes.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers.Text; +using System.Diagnostics; + +namespace System.Text.Json +{ + public sealed partial class Utf8JsonWriter + { + /// + /// Writes the raw bytes value as base 64 encoded JSON string as an element of a JSON array. + /// + /// The binary data to be written as a base 64 encoded JSON string element of a JSON array. + /// + /// The bytes are encoded before writing. + /// + /// + /// Thrown when the specified value is too large. + /// + /// + /// Thrown if this would result in an invalid JSON to be written (while validation is enabled). + /// + public void WriteBase64StringValue(ReadOnlySpan bytes) + { + JsonWriterHelper.ValidateBytes(bytes); + + WriteBase64ByOptions(bytes); + + SetFlagToAddListSeparatorBeforeNextItem(); + _tokenType = JsonTokenType.String; + } + + private void WriteBase64ByOptions(ReadOnlySpan bytes) + { + ValidateWritingValue(); + + if (Options.Indented) + { + WriteBase64Indented(bytes); + } + else + { + WriteBase64Minimized(bytes); + } + } + + // TODO: https://github.com/dotnet/corefx/issues/36958 + private void WriteBase64Minimized(ReadOnlySpan bytes) + { + int encodingLength = Base64.GetMaxEncodedToUtf8Length(bytes.Length); + + Debug.Assert(encodingLength < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileEscaping) - 3); + + // 2 quotes to surround the base-64 encoded string value, with escaping which can by up to 6x. + // Optionally, 1 list separator + int maxRequired = (encodingLength * JsonConstants.MaxExpansionFactorWhileEscaping) + 3; + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + output[BytesPending++] = JsonConstants.Quote; + + Base64EncodeAndWrite(bytes, output, encodingLength); + + output[BytesPending++] = JsonConstants.Quote; + } + + // TODO: https://github.com/dotnet/corefx/issues/36958 + private void WriteBase64Indented(ReadOnlySpan bytes) + { + int indent = Indentation; + Debug.Assert(indent <= 2 * JsonConstants.MaxWriterDepth); + + int encodingLength = Base64.GetMaxEncodedToUtf8Length(bytes.Length); + + Debug.Assert(encodingLength < (int.MaxValue / JsonConstants.MaxExpansionFactorWhileEscaping) - indent - 3 - s_newLineLength); + + // indentation + 2 quotes to surround the base-64 encoded string value, with escaping which can by up to 6x. + // Optionally, 1 list separator, and 1-2 bytes for new line + int maxRequired = indent + (encodingLength * JsonConstants.MaxExpansionFactorWhileEscaping) + 3 + s_newLineLength; + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + + if (_tokenType != JsonTokenType.None) + { + WriteNewLine(output); + } + + JsonWriterHelper.WriteIndentation(output.Slice(BytesPending), indent); + BytesPending += indent; + + output[BytesPending++] = JsonConstants.Quote; + + Base64EncodeAndWrite(bytes, output, encodingLength); + + output[BytesPending++] = JsonConstants.Quote; + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs index cc4986e..9c26cb3 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs @@ -2,7 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Buffers; +using System.Buffers.Text; using System.Diagnostics; +using System.Runtime.CompilerServices; namespace System.Text.Json { @@ -26,5 +29,41 @@ namespace System.Text.Json } } } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void Base64EncodeAndWrite(ReadOnlySpan bytes, Span output, int encodingLength) + { + byte[] outputText = null; + + Span encodedBytes = encodingLength <= JsonConstants.StackallocThreshold ? + stackalloc byte[encodingLength] : + (outputText = ArrayPool.Shared.Rent(encodingLength)); + + OperationStatus status = Base64.EncodeToUtf8(bytes, encodedBytes, out int consumed, out int written); + Debug.Assert(status == OperationStatus.Done); + Debug.Assert(consumed == bytes.Length); + + encodedBytes = encodedBytes.Slice(0, written); + Span destination = output.Slice(BytesPending); + + int firstEscapeIndexVal = encodedBytes.IndexOfAny(JsonConstants.Plus, JsonConstants.Slash); + if (firstEscapeIndexVal == -1) + { + Debug.Assert(destination.Length >= written); + encodedBytes.Slice(0, written).CopyTo(destination); + BytesPending += written; + } + else + { + Debug.Assert(destination.Length >= written * JsonConstants.MaxExpansionFactorWhileEscaping); + JsonWriterHelper.EscapeString(encodedBytes, destination, firstEscapeIndexVal, out written); + BytesPending += written; + } + + if (outputText != null) + { + ArrayPool.Shared.Return(outputText); + } + } } } diff --git a/src/libraries/System.Text.Json/tests/JsonDocumentTests.cs b/src/libraries/System.Text.Json/tests/JsonDocumentTests.cs index 36abf6b..c8d74f3 100644 --- a/src/libraries/System.Text.Json/tests/JsonDocumentTests.cs +++ b/src/libraries/System.Text.Json/tests/JsonDocumentTests.cs @@ -371,7 +371,7 @@ namespace System.Text.Json.Tests null, bytes => JsonDocument.Parse(new MemoryStream(Utf8Bom.Concat(bytes).ToArray()))); } - + [Theory] [MemberData(nameof(ReducedTestCases))] public static void ParseJson_SeekableStream_Async_WithBOM(bool compactData, TestCaseType type, string jsonString) @@ -540,12 +540,12 @@ namespace System.Text.Json.Tests } } - private static string PrintJson(this JsonDocument document, int sizeHint=0) + private static string PrintJson(this JsonDocument document, int sizeHint = 0) { return PrintJson(document.RootElement, sizeHint); } - private static string PrintJson(this JsonElement element, int sizeHint=0) + private static string PrintJson(this JsonElement element, int sizeHint = 0) { StringBuilder sb = new StringBuilder(sizeHint); DepthFirstAppend(sb, element); @@ -562,31 +562,31 @@ namespace System.Text.Json.Tests case JsonValueType.True: case JsonValueType.String: case JsonValueType.Number: - { - buf.Append(element.ToString()); - buf.Append(", "); - break; - } - case JsonValueType.Object: - { - foreach (JsonProperty prop in element.EnumerateObject()) { - buf.Append(prop.Name); + buf.Append(element.ToString()); buf.Append(", "); - DepthFirstAppend(buf, prop.Value); + break; } + case JsonValueType.Object: + { + foreach (JsonProperty prop in element.EnumerateObject()) + { + buf.Append(prop.Name); + buf.Append(", "); + DepthFirstAppend(buf, prop.Value); + } - break; - } + break; + } case JsonValueType.Array: - { - foreach (JsonElement child in element.EnumerateArray()) { - DepthFirstAppend(buf, child); - } + foreach (JsonElement child in element.EnumerateArray()) + { + DepthFirstAppend(buf, child); + } - break; - } + break; + } } } @@ -834,6 +834,8 @@ namespace System.Text.Json.Tests } Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); + Assert.Throws(() => root.TryGetBytesFromBase64(out byte[] bytes)); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -920,6 +922,7 @@ namespace System.Text.Json.Tests } Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -976,6 +979,7 @@ namespace System.Text.Json.Tests Assert.Equal(value, root.GetUInt64()); Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -1032,6 +1036,7 @@ namespace System.Text.Json.Tests Assert.Throws(() => root.GetUInt64()); Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -1220,6 +1225,7 @@ namespace System.Text.Json.Tests Assert.Throws(() => root.GetUInt64()); Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -1287,6 +1293,7 @@ namespace System.Text.Json.Tests Assert.Throws(() => root.GetUInt64()); Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -1338,6 +1345,8 @@ namespace System.Text.Json.Tests Assert.Throws(() => root.GetUInt64()); Assert.Throws(() => root.TryGetUInt64(out ulong _)); Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); + Assert.Throws(() => root.TryGetBytesFromBase64(out byte[] _)); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -1367,6 +1376,8 @@ namespace System.Text.Json.Tests Assert.Throws(() => root.GetUInt64()); Assert.Throws(() => root.TryGetUInt64(out ulong _)); Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); + Assert.Throws(() => root.TryGetBytesFromBase64(out byte[] _)); Assert.Throws(() => root.GetBoolean()); Assert.Throws(() => root.GetRawText()); @@ -1409,6 +1420,8 @@ namespace System.Text.Json.Tests Assert.Throws(() => root.GetUInt64()); Assert.Throws(() => root.TryGetUInt64(out ulong _)); Assert.Throws(() => root.GetString()); + Assert.Throws(() => root.GetBytesFromBase64()); + Assert.Throws(() => root.TryGetBytesFromBase64(out byte[] _)); Assert.Throws(() => root.GetDateTime()); Assert.Throws(() => root.GetDateTimeOffset()); Assert.Throws(() => root.GetGuid()); @@ -1467,6 +1480,107 @@ namespace System.Text.Json.Tests } } + [Fact] + public static void GetBase64String_BadUtf8() + { + // The Arabic ligature Lam-Alef (U+FEFB) (which happens to, as a standalone, mean "no" in English) + // is UTF-8 EF BB BB. So let's leave out a BB and put it in quotes. + using (JsonDocument doc = JsonDocument.Parse(new byte[] { 0x22, 0xEF, 0xBB, 0x22 })) + { + JsonElement root = doc.RootElement; + + Assert.Equal(JsonValueType.String, root.Type); + Assert.Throws(() => root.GetBytesFromBase64()); + Assert.False(root.TryGetBytesFromBase64(out byte[] value)); + Assert.Null(value); + } + } + + [Fact] + public static void GetBase64Unescapes() + { + string jsonString = "\"\\u0031234\""; // equivalent to "\"1234\"" + + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + + using (JsonDocument doc = JsonDocument.Parse(dataUtf8)) + { + byte[] expected = Convert.FromBase64String("1234"); // new byte[3] { 215, 109, 248 } + + Assert.Equal(expected, doc.RootElement.GetBytesFromBase64()); + + Assert.True(doc.RootElement.TryGetBytesFromBase64(out byte[] value)); + Assert.Equal(expected, value); + } + } + + [Theory] + [InlineData("\"ABC=\"")] + [InlineData("\"AB+D\"")] + [InlineData("\"ABCD\"")] + [InlineData("\"ABC/\"")] + [InlineData("\"++++\"")] + [InlineData(null)] // Large randomly generated string + public static void ReadBase64String(string jsonString) + { + if (jsonString == null) + { + var random = new Random(42); + var charArray = new char[502]; + charArray[0] = '"'; + for (int i = 1; i < charArray.Length; i++) + { + charArray[i] = (char)random.Next('A', 'Z'); // ASCII values (between 65 and 90) that constitute valid base 64 string. + } + charArray[charArray.Length - 1] = '"'; + jsonString = new string(charArray); + } + + byte[] expected = Convert.FromBase64String(jsonString.AsSpan(1, jsonString.Length - 2).ToString()); + + using (JsonDocument doc = JsonDocument.Parse(jsonString)) + { + Assert.Equal(expected, doc.RootElement.GetBytesFromBase64()); + + Assert.True(doc.RootElement.TryGetBytesFromBase64(out byte[] value)); + Assert.Equal(expected, value); + } + } + + [Theory] + [InlineData("\"ABC===\"")] + [InlineData("\"ABC\"")] + [InlineData("\"ABC!\"")] + [InlineData(null)] // Large randomly generated string + public static void InvalidBase64(string jsonString) + { + if (jsonString == null) + { + var random = new Random(42); + var charArray = new char[500]; + charArray[0] = '"'; + for (int i = 1; i < charArray.Length; i++) + { + charArray[i] = (char)random.Next('?', '\\'); // ASCII values (between 63 and 91) that don't need to be escaped. + } + + charArray[256] = '\\'; + charArray[257] = '"'; + charArray[charArray.Length - 1] = '"'; + jsonString = new string(charArray); + } + + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + + using (JsonDocument doc = JsonDocument.Parse(dataUtf8)) + { + Assert.False(doc.RootElement.TryGetBytesFromBase64(out byte[] value)); + Assert.Null(value); + + Assert.Throws(() => doc.RootElement.GetBytesFromBase64()); + } + } + [Theory] [InlineData(" { \"hi\": \"there\" }")] [InlineData(" { \n\n\n\n } ")] @@ -1878,8 +1992,8 @@ namespace System.Text.Json.Tests public static void GetRawText() { const string json = - // Don't let there be a newline before the first embedded quote, - // because the index would change across CRLF vs LF compile environments. +// Don't let there be a newline before the first embedded quote, +// because the index would change across CRLF vs LF compile environments. @"{ "" weird property name"" : { @@ -2925,7 +3039,7 @@ namespace System.Text.Json.Tests ReadOnlyMemory data, int segmentCount, in JsonReaderState state, - bool isFinalBlock=false) + bool isFinalBlock = false) { if (segmentCount == 0) { diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs index 642cb87..6468872 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs @@ -495,6 +495,22 @@ namespace System.Text.Json.Tests try { + byte[] value = json.GetBytesFromBase64(); + Assert.True(false, "Expected GetBytesFromBase64 to throw InvalidOperationException due to mismatch token type."); + } + catch (InvalidOperationException) + { } + + try + { + json.TryGetBytesFromBase64(out byte[] value); + Assert.True(false, "Expected TryGetBytesFromBase64 to throw InvalidOperationException due to mismatch token type."); + } + catch (InvalidOperationException) + { } + + try + { DateTime value = json.GetDateTime(); Assert.True(false, "Expected GetDateTime to throw InvalidOperationException due to mismatched token type."); } @@ -813,6 +829,170 @@ namespace System.Text.Json.Tests } } + [Fact] + public static void GetBase64Unescapes() + { + string jsonString = "\"\\u0031234\""; // equivalent to "\"1234\"" + + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + + var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state: default); + Assert.True(json.Read()); + + byte[] expected = Convert.FromBase64String("1234"); // new byte[3] { 215, 109, 248 } + + byte[] value = json.GetBytesFromBase64(); + Assert.Equal(expected, value); + Assert.True(json.TryGetBytesFromBase64(out value)); + Assert.Equal(expected, value); + } + + [Theory] + [InlineData("\"ABC=\"")] + [InlineData("\"AB+D\"")] + [InlineData("\"ABCD\"")] + [InlineData("\"ABC/\"")] + [InlineData("\"++++\"")] + [InlineData(null)] // Large randomly generated string + public static void ValidBase64(string jsonString) + { + if (jsonString == null) + { + var random = new Random(42); + var charArray = new char[502]; + charArray[0] = '"'; + for (int i = 1; i < charArray.Length; i++) + { + charArray[i] = (char)random.Next('A', 'Z'); // ASCII values (between 65 and 90) that constitute valid base 64 string. + } + charArray[charArray.Length - 1] = '"'; + jsonString = new string(charArray); + } + + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + + var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state: default); + Assert.True(json.Read()); + + byte[] expected = Convert.FromBase64String(jsonString.AsSpan(1, jsonString.Length - 2).ToString()); + + byte[] value = json.GetBytesFromBase64(); + Assert.Equal(expected, value); + Assert.True(json.TryGetBytesFromBase64(out value)); + Assert.Equal(expected, value); + } + + [Theory] + [InlineData("\"ABC===\"")] + [InlineData("\"ABC\"")] + [InlineData("\"ABC!\"")] + [InlineData(null)] // Large randomly generated string + public static void InvalidBase64(string jsonString) + { + if (jsonString == null) + { + var random = new Random(42); + var charArray = new char[500]; + charArray[0] = '"'; + for (int i = 1; i < charArray.Length; i++) + { + charArray[i] = (char)random.Next('?', '\\'); // ASCII values (between 63 and 91) that don't need to be escaped. + } + + charArray[256] = '\\'; + charArray[257] = '"'; + charArray[charArray.Length - 1] = '"'; + jsonString = new string(charArray); + } + + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + + var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state: default); + Assert.True(json.Read()); + Assert.False(json.TryGetBytesFromBase64(out byte[] value)); + Assert.Null(value); + + try + { + byte[] val = json.GetBytesFromBase64(); + Assert.True(false, "Expected InvalidOperationException when trying to decode base 64 string for invalid UTF-16 JSON text."); + } + catch (FormatException) { } + } + + [Theory] + [InlineData("\"a\\uDD1E\"")] + [InlineData("\"a\\uDD1Eb\"")] + [InlineData("\"a\\uD834\"")] + [InlineData("\"a\\uD834\\u0030\"")] + [InlineData("\"a\\uD834\\uD834\"")] + [InlineData("\"a\\uD834b\"")] + [InlineData("\"a\\uDD1E\\uD834b\"")] + [InlineData("\"a\\\\uD834\\uDD1Eb\"")] + [InlineData("\"a\\uDD1E\\\\uD834b\"")] + public static void TestingGetBase64InvalidUTF16(string jsonString) + { + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state); + + Assert.True(json.Read()); + Assert.Equal(JsonTokenType.String, json.TokenType); + try + { + byte[] val = json.GetBytesFromBase64(); + Assert.True(false, "Expected InvalidOperationException when trying to decode base 64 string for invalid UTF-16 JSON text."); + } + catch (InvalidOperationException) { } + + try + { + json.TryGetBytesFromBase64(out byte[] val); + Assert.True(false, "Expected InvalidOperationException when trying to decode base 64 string for invalid UTF-16 JSON text."); + } + catch (InvalidOperationException) { } + } + } + + [Theory] + [MemberData(nameof(InvalidUTF8Strings))] + public static void TestingGetBase64InvalidUTF8(byte[] dataUtf8) + { + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state); + + // It is expected that the Utf8JsonReader won't throw an exception here + Assert.True(json.Read()); + Assert.Equal(JsonTokenType.String, json.TokenType); + + while (json.Read()) + ; + + json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state); + + while (json.Read()) + { + if (json.TokenType == JsonTokenType.String) + { + try + { + byte[] val = json.GetBytesFromBase64(); + Assert.True(false, "Expected InvalidOperationException when trying to decode base 64 string for invalid UTF-8 JSON text."); + } + catch (FormatException) { } + + Assert.False(json.TryGetBytesFromBase64(out byte[] value)); + Assert.Null(value); + } + } + } + } + [Theory] [MemberData(nameof(GetCommentTestData))] public static void TestingGetComment(string jsonData, string expected) diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs index 2d1e30f..9a046b3 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs @@ -1408,6 +1408,50 @@ namespace System.Text.Json.Tests Assert.Throws(() => jsonUtf8.WriteStartArray(key)); } + [ConditionalTheory(nameof(IsX64))] + [OuterLoop] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public void WritingTooLargeBase64Bytes(bool formatted, bool skipValidation) + { + byte[] value; + + try + { + value = new byte[200_000_000]; + } + catch (OutOfMemoryException) + { + return; + } + + value.AsSpan().Fill(255); + + var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation }; + var output = new ArrayBufferWriter(1024); + + var jsonUtf8 = new Utf8JsonWriter(output, options); + Assert.Throws(() => jsonUtf8.WriteBase64StringValue(value)); + + jsonUtf8 = new Utf8JsonWriter(output, options); + jsonUtf8.WriteStartObject(); + Assert.Throws(() => jsonUtf8.WriteBase64String("foo", value)); + + jsonUtf8 = new Utf8JsonWriter(output, options); + jsonUtf8.WriteStartObject(); + Assert.Throws(() => jsonUtf8.WriteBase64String(Encoding.UTF8.GetBytes("foo"), value)); + + jsonUtf8 = new Utf8JsonWriter(output, options); + jsonUtf8.WriteStartObject(); + Assert.Throws(() => jsonUtf8.WriteBase64String("foo".AsSpan(), value)); + + jsonUtf8 = new Utf8JsonWriter(output, options); + jsonUtf8.WriteStartObject(); + Assert.Throws(() => jsonUtf8.WriteBase64String(JsonEncodedText.Encode("foo"), value)); + } + [Theory] [InlineData(true, true)] [InlineData(true, false)] @@ -1703,6 +1747,171 @@ namespace System.Text.Json.Tests [InlineData(true, false)] [InlineData(false, true)] [InlineData(false, false)] + public void WriteBase64String(bool formatted, bool skipValidation) + { + string propertyName = "message"; + byte[] value = { 1, 2, 3, 4, 5 }; + string expectedStr = GetBase64ExpectedString(prettyPrint: formatted, propertyName, value); + + JsonEncodedText encodedPropertyName = JsonEncodedText.Encode(propertyName); + + byte[] utf8PropertyName = Encoding.UTF8.GetBytes("message"); + + var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation }; + + for (int i = 0; i < 4; i++) + { + var output = new ArrayBufferWriter(32); + var jsonUtf8 = new Utf8JsonWriter(output, options); + + jsonUtf8.WriteStartObject(); + + switch (i) + { + case 0: + jsonUtf8.WriteBase64String(propertyName, value); + jsonUtf8.WriteBase64String(propertyName, value); + break; + case 1: + jsonUtf8.WriteBase64String(propertyName.AsSpan(), value); + jsonUtf8.WriteBase64String(propertyName.AsSpan(), value); + break; + case 2: + jsonUtf8.WriteBase64String(utf8PropertyName, value); + jsonUtf8.WriteBase64String(utf8PropertyName, value); + break; + case 3: + jsonUtf8.WriteBase64String(encodedPropertyName, value); + jsonUtf8.WriteBase64String(encodedPropertyName, value); + break; + } + + jsonUtf8.WriteEndObject(); + jsonUtf8.Flush(); + + AssertContents(expectedStr, output); + } + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public void WriteBase64StringEscaped(bool formatted, bool skipValidation) + { + string propertyName = "mess> propertyNameSpan = propertyName.AsSpan(); + ReadOnlySpan propertyNameSpanUtf8 = Encoding.UTF8.GetBytes(propertyName); + JsonEncodedText encodedPropertyName = JsonEncodedText.Encode(propertyName); + + for (int i = 0; i < 4; i++) + { + var output = new ArrayBufferWriter(32); + var jsonUtf8 = new Utf8JsonWriter(output, options); + + jsonUtf8.WriteStartObject(); + + switch (i) + { + case 0: + jsonUtf8.WriteBase64String(propertyName, value); + jsonUtf8.WriteBase64String(propertyName, value); + break; + case 1: + jsonUtf8.WriteBase64String(propertyNameSpan, value); + jsonUtf8.WriteBase64String(propertyNameSpan, value); + break; + case 2: + jsonUtf8.WriteBase64String(propertyNameSpanUtf8, value); + jsonUtf8.WriteBase64String(propertyNameSpanUtf8, value); + break; + case 3: + jsonUtf8.WriteBase64String(encodedPropertyName, value); + jsonUtf8.WriteBase64String(encodedPropertyName, value); + break; + } + + jsonUtf8.WriteEndObject(); + jsonUtf8.Flush(); + + AssertContents(expectedStr, output); + } + + // Verify that escaping does not change the input strings/spans. + Assert.Equal("mess>(10); + var jsonUtf8 = new Utf8JsonWriter(output, options); + + jsonUtf8.WriteStartObject(); + + Assert.Equal(0, jsonUtf8.BytesCommitted); + Assert.Equal(1, jsonUtf8.BytesPending); + + jsonUtf8.WriteBase64String("message", new byte[] { 201, 153, 199 }); + + Assert.Equal(0, jsonUtf8.BytesCommitted); + if (formatted) + Assert.Equal(17 + 2 + Environment.NewLine.Length + 1, jsonUtf8.BytesPending); // new lines, indentation, white space + else + Assert.Equal(17, jsonUtf8.BytesPending); + + jsonUtf8.Flush(); + + if (formatted) + Assert.Equal(17 + 2 + Environment.NewLine.Length + 1, jsonUtf8.BytesCommitted); // new lines, indentation, white space + else + Assert.Equal(17, jsonUtf8.BytesCommitted); + + Assert.Equal(0, jsonUtf8.BytesPending); + + jsonUtf8.WriteBase64String("message", new byte[] { 201, 153, 199 }); + jsonUtf8.WriteEndObject(); + + if (formatted) + Assert.Equal(17 + 2 + Environment.NewLine.Length + 1, jsonUtf8.BytesCommitted); + else + Assert.Equal(17, jsonUtf8.BytesCommitted); + + if (formatted) + Assert.Equal(18 + 2 + (2 * Environment.NewLine.Length) + 1, jsonUtf8.BytesPending); // new lines, indentation, white space + else + Assert.Equal(18, jsonUtf8.BytesPending); + + jsonUtf8.Flush(); + + if (formatted) + Assert.Equal(35 + (2 * 2) + (3 * Environment.NewLine.Length) + (1 * 2), jsonUtf8.BytesCommitted); // new lines, indentation, white space + else + Assert.Equal(35, jsonUtf8.BytesCommitted); + + Assert.Equal(0, jsonUtf8.BytesPending); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] public void WriteInvalidPartialJson(bool formatted, bool skipValidation) { var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation }; @@ -1736,6 +1945,111 @@ namespace System.Text.Json.Tests [InlineData(true, false)] [InlineData(false, true)] [InlineData(false, false)] + public void WriteInvalidBase64(bool formatted, bool skipValidation) + { + { + var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation }; + var output = new ArrayBufferWriter(10); + using var jsonUtf8 = new Utf8JsonWriter(output, options); + + jsonUtf8.WriteStartObject(); + + Assert.Equal(0, jsonUtf8.BytesCommitted); + Assert.Equal(1, jsonUtf8.BytesPending); + + jsonUtf8.Flush(); + + Assert.Equal(1, jsonUtf8.BytesCommitted); + Assert.Equal(0, jsonUtf8.BytesPending); + + if (skipValidation) + { + jsonUtf8.WriteBase64StringValue(new byte[] { 1, 2 }); + jsonUtf8.WriteEndArray(); + } + else + { + Assert.Throws(() => jsonUtf8.WriteBase64StringValue(new byte[] { 1, 2 })); + Assert.Throws(() => jsonUtf8.WriteEndArray()); + } + } + { + var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation }; + var output = new ArrayBufferWriter(10); + using var jsonUtf8 = new Utf8JsonWriter(output, options); + + jsonUtf8.WriteStartArray(); + + Assert.Equal(0, jsonUtf8.BytesCommitted); + Assert.Equal(1, jsonUtf8.BytesPending); + + jsonUtf8.Flush(); + + Assert.Equal(1, jsonUtf8.BytesCommitted); + Assert.Equal(0, jsonUtf8.BytesPending); + + if (skipValidation) + { + jsonUtf8.WriteBase64String("foo", new byte[] { 1, 2 }); + jsonUtf8.WriteEndObject(); + } + else + { + Assert.Throws(() => jsonUtf8.WriteBase64String("foo", new byte[] { 1, 2 })); + Assert.Throws(() => jsonUtf8.WriteEndObject()); + } + } + } + + [Fact] + public void WriteBase64Escapes() + { + var output = new ArrayBufferWriter(10); + using var jsonUtf8 = new Utf8JsonWriter(output); + + var bytes = new byte[3] { 0xFB, 0xEF, 0xBE }; + jsonUtf8.WriteBase64StringValue(bytes); + + jsonUtf8.Flush(); + + AssertContents("\"\\u002b\\u002b\\u002b\\u002b\"", output); + } + + [Fact] + public void WriteBase64EscapesLarge() + { + var output = new ArrayBufferWriter(10); + using var jsonUtf8 = new Utf8JsonWriter(output); + + var bytes = new byte[200]; + + bytes.AsSpan().Fill(100); + bytes[4] = 0xFB; + bytes[5] = 0xEF; + bytes[6] = 0xBE; + bytes[15] = 0; + bytes[16] = 0x10; + bytes[17] = 0xBF; + + jsonUtf8.WriteBase64StringValue(bytes); + + jsonUtf8.Flush(); + + var builder = new StringBuilder(); + builder.Append("\"ZGRkZPvvvmRkZGRkZGRkABC\\u002f"); + for (int i = 0; i < 60; i++) + { + builder.Append("ZGRk"); + } + builder.Append("ZGQ=\""); + AssertContents(builder.ToString(), output); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] public void WriteInvalidDepthPartial(bool formatted, bool skipValidation) { { @@ -2083,7 +2397,7 @@ namespace System.Text.Json.Tests { var propertyArray = new char[128]; - char[] specialCases = { '+', '`', (char)0x7F }; + char[] specialCases = { '+', '`', (char)0x7F, '/' }; for (int i = 0; i < propertyArray.Length; i++) { if (Array.IndexOf(specialCases, (char)i) != -1) @@ -3939,6 +4253,29 @@ namespace System.Text.Json.Tests return Encoding.UTF8.GetString(ms.ToArray()); } + private static string GetBase64ExpectedString(bool prettyPrint, string propertyName, byte[] value) + { + var ms = new MemoryStream(); + TextWriter streamWriter = new StreamWriter(ms, new UTF8Encoding(false), 1024, true); + + var json = new JsonTextWriter(streamWriter) + { + Formatting = prettyPrint ? Formatting.Indented : Formatting.None, + StringEscapeHandling = StringEscapeHandling.EscapeHtml + }; + + json.WriteStartObject(); + json.WritePropertyName(propertyName); + json.WriteValue(value); + json.WritePropertyName(propertyName); + json.WriteValue(value); + json.WriteEnd(); + + json.Flush(); + + return Encoding.UTF8.GetString(ms.ToArray()); + } + private static string GetCommentExpectedString(bool prettyPrint, string comment) { var ms = new MemoryStream(); -- 2.7.4