<data name="ZeroDepthAtEnd" xml:space="preserve">
<value>Expected depth to be zero at the end of the JSON payload. There is an open JSON object or array that should be closed.</value>
</data>
- <data name="DeserializeDataRemaining" xml:space="preserve">
- <value>The provided data of length {0} has remaining bytes {1}.</value>
- </data>
<data name="DeserializeUnableToConvertValue" xml:space="preserve">
<value>The JSON value could not be converted to {0}.</value>
</data>
key = MemoryMarshal.Read<ulong>(propertyName);
// Max out the length byte.
- // The comparison logic tests for equality against the full contents instead of just
- // the key if the property name length is >7.
+ // This will cause the comparison logic to always test for equality against the full contents
+ // when the first 7 bytes are the same.
key |= 0xFF00000000000000;
+
+ // It is also possible to include the length up to 0xFF in order to prevent false positives
+ // when the first 7 bytes match but a different length (up to 0xFF). However the extra logic
+ // slows key generation in the majority of cases:
+ // key &= 0x00FFFFFFFFFFFFFF;
+ // key |= (ulong) 7 << Math.Max(length, 0xFF);
}
else if (length > 3)
{
if (length == 7)
{
- key |= (ulong) propertyName[6] << (6 * BitsInByte)
- | (ulong) propertyName[5] << (5 * BitsInByte)
- | (ulong) propertyName[4] << (4 * BitsInByte)
- | (ulong) 7 << (7 * BitsInByte);
+ key |= (ulong)propertyName[6] << (6 * BitsInByte)
+ | (ulong)propertyName[5] << (5 * BitsInByte)
+ | (ulong)propertyName[4] << (4 * BitsInByte)
+ | (ulong)7 << (7 * BitsInByte);
}
else if (length == 6)
{
- key |= (ulong) propertyName[5] << (5 * BitsInByte)
- | (ulong) propertyName[4] << (4 * BitsInByte)
- | (ulong) 6 << (7 * BitsInByte);
+ key |= (ulong)propertyName[5] << (5 * BitsInByte)
+ | (ulong)propertyName[4] << (4 * BitsInByte)
+ | (ulong)6 << (7 * BitsInByte);
}
else if (length == 5)
{
- key |= (ulong) propertyName[4] << (4 * BitsInByte)
- | (ulong) 5 << (7 * BitsInByte);
+ key |= (ulong)propertyName[4] << (4 * BitsInByte)
+ | (ulong)5 << (7 * BitsInByte);
}
else
{
- key |= (ulong) 4 << (7 * BitsInByte);
+ key |= (ulong)4 << (7 * BitsInByte);
}
}
else if (length > 1)
if (length == 3)
{
- key |= (ulong) propertyName[2] << (2 * BitsInByte)
- | (ulong) 3 << (7 * BitsInByte);
+ key |= (ulong)propertyName[2] << (2 * BitsInByte)
+ | (ulong)3 << (7 * BitsInByte);
}
else
{
- key |= (ulong) 2 << (7 * BitsInByte);
+ key |= (ulong)2 << (7 * BitsInByte);
}
}
else if (length == 1)
{
key = propertyName[0]
- | (ulong) 1 << (7 * BitsInByte);
+ | (ulong)1 << (7 * BitsInByte);
}
else
{
Debug.Assert(ShouldDeserialize);
JsonPropertyInfo propertyInfo;
- if (ElementClassInfo != null && (propertyInfo = ElementClassInfo.PolicyProperty) != null)
+ JsonClassInfo elementClassInfo = ElementClassInfo;
+ if (elementClassInfo != null && (propertyInfo = elementClassInfo.PolicyProperty) != null)
{
// Forward the setter to the value-based JsonPropertyInfo.
propertyInfo.ReadEnumerable(tokenType, ref state, ref reader);
state.Current.CollectionPropertyInitialized = true;
- // We should not be processing custom converters here.
+ // We should not be processing custom converters here (converters are of ClassType.Value).
Debug.Assert(state.Current.JsonClassInfo.ClassType != ClassType.Value);
// Set or replace the existing enumerable value.
state.Current.CollectionPropertyInitialized = true;
ClassType classType = state.Current.JsonClassInfo.ClassType;
- if (classType == ClassType.Value &&
- jsonPropertyInfo.ElementClassInfo.Type != typeof(object) &&
- jsonPropertyInfo.ElementClassInfo.Type != typeof(JsonElement))
+ if (classType == ClassType.Value)
{
- ThrowHelper.ThrowJsonException_DeserializeUnableToConvertValue(state.Current.JsonClassInfo.Type);
+ Type elementClassInfoType = jsonPropertyInfo.ElementClassInfo.Type;
+ if (elementClassInfoType != typeof(object) && elementClassInfoType != typeof(JsonElement))
+ {
+ ThrowHelper.ThrowJsonException_DeserializeUnableToConvertValue(state.Current.JsonClassInfo.Type);
+ }
}
JsonClassInfo classInfo = state.Current.JsonClassInfo;
Debug.Assert(state.Current.ReturnValue != default || state.Current.TempDictionaryValues != default);
Debug.Assert(state.Current.JsonClassInfo != default);
- if (state.Current.IsProcessingDictionaryOrIDictionaryConstructible() &&
+ bool isProcessingDictObject = state.Current.IsProcessingDictionaryOrIDictionaryConstructibleObject();
+ if ((isProcessingDictObject || state.Current.IsProcessingDictionaryOrIDictionaryConstructibleProperty()) &&
state.Current.JsonClassInfo.DataExtensionProperty != state.Current.JsonPropertyInfo)
{
- if (state.Current.IsProcessingObject(ClassType.Dictionary | ClassType.IDictionaryConstructible))
+ if (isProcessingDictObject)
{
state.Current.JsonPropertyInfo = state.Current.JsonClassInfo.PolicyProperty;
}
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
+using System.Diagnostics;
+
namespace System.Text.Json
{
public static partial class JsonSerializer
var reader = new Utf8JsonReader(utf8Json, isFinalBlock: true, readerState);
object result = ReadCore(returnType, options, ref reader);
- if (reader.BytesConsumed != utf8Json.Length)
- {
- ThrowHelper.ThrowJsonException_DeserializeDataRemaining(utf8Json.Length, utf8Json.Length - reader.BytesConsumed);
- }
+ // The reader should have thrown if we have remaining bytes.
+ Debug.Assert(reader.BytesConsumed == utf8Json.Length);
return result;
}
ArrayPool<byte>.Shared.Return(buffer);
}
- if (bytesInBuffer != 0)
- {
- ThrowHelper.ThrowJsonException_DeserializeDataRemaining(totalBytesRead, bytesInBuffer);
- }
+ // The reader should have thrown if we have remaining bytes.
+ Debug.Assert(bytesInBuffer == 0);
return (TValue)readStack.Current.ReturnValue;
}
// See the LICENSE file in the project root for more information.
using System.Buffers;
+using System.Diagnostics;
namespace System.Text.Json
{
public static partial class JsonSerializer
{
- // The maximum length of a byte array before we ask for the actual transcoded byte size.
- // The "int.MaxValue / 2" is used because max byte[] length is a bit less than int.MaxValue
- // and because we don't want to allocate such a large buffer if not necessary.
- private const int MaxArrayLengthBeforeCalculatingSize = int.MaxValue / 2 / JsonConstants.MaxExpansionFactorWhileTranscoding;
-
/// <summary>
/// Parse the text representing a single JSON value into a <typeparamref name="TValue"/>.
/// </summary>
/// </remarks>
public static object Deserialize(string json, Type returnType, JsonSerializerOptions options = null)
{
+ const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024;
+
if (json == null)
{
throw new ArgumentNullException(nameof(json));
object result;
byte[] tempArray = null;
- int maxBytes;
-
- // For performance, avoid asking for the actual byte count unless necessary.
- if (json.Length > MaxArrayLengthBeforeCalculatingSize)
- {
- // Get the actual byte count in order to handle large input.
- maxBytes = JsonReaderHelper.GetUtf8ByteCount(json.AsSpan());
- }
- else
- {
- maxBytes = json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding;
- }
-
- Span<byte> utf8 = maxBytes <= JsonConstants.StackallocThreshold ?
- stackalloc byte[maxBytes] :
- (tempArray = ArrayPool<byte>.Shared.Rent(maxBytes));
+ // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold.
+ Span<byte> utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ?
+ // Use a pooled alloc.
+ tempArray = ArrayPool<byte>.Shared.Rent(json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) :
+ // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation)
+ // and by using a normal alloc we can avoid the Clear().
+ new byte[JsonReaderHelper.GetUtf8ByteCount(json.AsSpan())];
try
{
var reader = new Utf8JsonReader(utf8, isFinalBlock: true, readerState);
result = ReadCore(returnType, options, ref reader);
- if (reader.BytesConsumed != actualByteCount)
- {
- ThrowHelper.ThrowJsonException_DeserializeDataRemaining(
- actualByteCount, actualByteCount - reader.BytesConsumed);
- }
+ // The reader should have thrown if we have remaining bytes.
+ Debug.Assert(reader.BytesConsumed == actualByteCount);
}
finally
{
ReadCore(options, ref newReader, ref readStack);
- if (newReader.BytesConsumed != length)
- {
- ThrowHelper.ThrowJsonException_DeserializeDataRemaining(length, length - newReader.BytesConsumed);
- }
+ // The reader should have thrown if we have remaining bytes.
+ Debug.Assert(newReader.BytesConsumed == length);
}
catch (JsonException)
{
using System.Diagnostics;
using System.Runtime.CompilerServices;
+using System.Text.Json.Serialization;
namespace System.Text.Json
{
}
state.Current.WriteObjectOrArrayStart(ClassType.Object, writer, options);
- state.Current.PropertyEnumeratorActive = true;
state.Current.MoveToNextProperty = true;
}
}
// Determine if we are done enumerating properties.
- if (state.Current.PropertyEnumeratorActive)
+ if (state.Current.ExtensionDataStatus != ExtensionDataWriteStatus.Finished)
{
// If ClassType.Unknown at this point, we are typeof(object) which should not have any properties.
Debug.Assert(state.Current.JsonClassInfo.ClassType != ClassType.Unknown);
{
state.Pop();
}
- else
- {
- state.Current.EndObject();
- }
return true;
}
}
/// <summary>
+ /// Is the current object a Dictionary or IDictionaryConstructible.
+ /// </summary>
+ public bool IsProcessingDictionaryOrIDictionaryConstructibleObject()
+ {
+ return IsProcessingObject(ClassType.Dictionary | ClassType.IDictionaryConstructible);
+ }
+
+ /// <summary>
+ /// Is the current property a Dictionary or IDictionaryConstructible.
+ /// </summary>
+ public bool IsProcessingDictionaryOrIDictionaryConstructibleProperty()
+ {
+ return IsProcessingProperty(ClassType.Dictionary | ClassType.IDictionaryConstructible);
+ }
+
+ /// <summary>
/// Is the current object or property a Dictionary or IDictionaryConstructible.
/// </summary>
public bool IsProcessingDictionaryOrIDictionaryConstructible()
if (jsonPropertyInfo.EnumerableConverter != null)
{
IList converterList;
- if (jsonPropertyInfo.ElementClassInfo.ClassType == ClassType.Value)
+ JsonClassInfo elementClassInfo = jsonPropertyInfo.ElementClassInfo;
+ if (elementClassInfo.ClassType == ClassType.Value)
{
- converterList = jsonPropertyInfo.ElementClassInfo.PolicyProperty.CreateConverterList();
+ converterList = elementClassInfo.PolicyProperty.CreateConverterList();
}
else
{
public bool MoveToNextProperty;
// The current property.
- public bool PropertyEnumeratorActive;
public int PropertyEnumeratorIndex;
public ExtensionDataWriteStatus ExtensionDataStatus;
public JsonPropertyInfo JsonPropertyInfo;
public void Reset()
{
CurrentValue = null;
- EndObject();
- }
-
- public void EndObject()
- {
CollectionEnumerator = null;
ExtensionDataStatus = ExtensionDataWriteStatus.NotStarted;
IsIDictionaryConstructible = false;
JsonClassInfo = null;
PropertyEnumeratorIndex = 0;
- PropertyEnumeratorActive = false;
PopStackOnEndCollection = false;
PopStackOnEndObject = false;
StartObjectWritten = false;
+
EndProperty();
}
{
EndProperty();
- if (PropertyEnumeratorActive)
+ int maxPropertyIndex = JsonClassInfo.PropertyCacheArray.Length;
+
+ ++PropertyEnumeratorIndex;
+ if (PropertyEnumeratorIndex >= maxPropertyIndex)
{
- int len = JsonClassInfo.PropertyCacheArray.Length;
- if (PropertyEnumeratorIndex < len)
+ if (PropertyEnumeratorIndex > maxPropertyIndex)
{
- if ((PropertyEnumeratorIndex == len - 1) && JsonClassInfo.DataExtensionProperty != null)
- {
- ExtensionDataStatus = ExtensionDataWriteStatus.Writing;
- }
-
- PropertyEnumeratorIndex++;
- PropertyEnumeratorActive = true;
+ ExtensionDataStatus = ExtensionDataWriteStatus.Finished;
}
- else
+ else if (JsonClassInfo.DataExtensionProperty != null)
{
- PropertyEnumeratorActive = false;
+ ExtensionDataStatus = ExtensionDataWriteStatus.Writing;
}
}
- else
- {
- ExtensionDataStatus = ExtensionDataWriteStatus.Finished;
- }
}
}
}
throw new InvalidOperationException(SR.Format(SR.SerializerDictionaryKeyNull, policyType));
}
- public static void ThrowJsonException_DeserializeDataRemaining(long length, long bytesRemaining)
- {
- throw new JsonException(SR.Format(SR.DeserializeDataRemaining, length, bytesRemaining), path: null, lineNumber: null, bytePositionInLine: null);
- }
-
[MethodImpl(MethodImplOptions.NoInlining)]
public static void ReThrowWithPath(in ReadStack readStack, JsonReaderException ex)
{
string message = ex.Message;
- // If the message is empty or contains EmbedPathInfoFlag then append the Path, LineNumber and BytePositionInLine.
if (string.IsNullOrEmpty(message))
{
// Use a default message.
string path = writeStack.PropertyPath();
ex.Path = path;
- // If the message is empty, use a default message with the Path.
string message = ex.Message;
if (string.IsNullOrEmpty(message))
{
+ // Use a default message.
message = SR.Format(SR.SerializeUnableToSerialize);
ex.AppendPathInformation = true;
}
=> writer.WriteStringValue(value);
}
}
+
+ private class ConverterReturningNull : JsonConverter<Customer>
+ {
+ public override Customer Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+ {
+ Assert.Equal(JsonTokenType.StartObject, reader.TokenType);
+
+ bool rc = reader.Read();
+ Assert.True(rc);
+
+ Assert.Equal(JsonTokenType.EndObject, reader.TokenType);
+
+ return null;
+ }
+
+ public override void Write(Utf8JsonWriter writer, Customer value, JsonSerializerOptions options)
+ {
+ throw new NotSupportedException();
+ }
+ }
+
+ [Fact]
+ public static void VerifyConverterWithTrailingWhitespace()
+ {
+ string json = "{} ";
+
+ var options = new JsonSerializerOptions();
+ options.Converters.Add(new ConverterReturningNull());
+
+ byte[] utf8 = Encoding.UTF8.GetBytes(json);
+
+ // The serializer will finish reading the whitespace and no exception will be thrown.
+ Customer c = JsonSerializer.Deserialize<Customer>(utf8, options);
+
+ Assert.Null(c);
+ }
}
}
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
-using System.Collections;
using System.Collections.Generic;
+using System.IO;
using Xunit;
namespace System.Text.Json.Serialization.Tests
Assert.Equal(1, parsedObject.ParsedSubMixedTypeParsedClass.ParsedDictionary.Count);
Assert.Equal(18, parsedObject.ParsedSubMixedTypeParsedClass.ParsedDictionary["Key1"]);
}
+
+ private class POCO { }
+
+ [Theory]
+ [InlineData("{}{}")]
+ [InlineData("{}1")]
+ [InlineData("{}/**/")]
+ [InlineData("{} /**/")]
+ public static void TooMuchJsonFails(string json)
+ {
+ byte[] jsonBytes = Encoding.UTF8.GetBytes(json);
+
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<POCO>(json));
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<POCO>(jsonBytes));
+ Assert.Throws<JsonException>(() => JsonSerializer.DeserializeAsync<POCO>(new MemoryStream(jsonBytes)).Result);
+
+ // Using a reader directly doesn't throw since it stops once POCO is read.
+ Utf8JsonReader reader = new Utf8JsonReader(jsonBytes);
+ JsonSerializer.Deserialize<POCO>(ref reader);
+
+ Assert.True(reader.BytesConsumed > 0);
+ Assert.NotEqual(jsonBytes.Length, reader.BytesConsumed);
+ }
+
+ [Theory]
+ [InlineData("{")]
+ [InlineData(@"{""")]
+ [InlineData(@"{""a""")]
+ [InlineData(@"{""a"":")]
+ [InlineData(@"{""a"":1")]
+ [InlineData(@"{""a"":1,")]
+ public static void TooLittleJsonFails(string json)
+ {
+ byte[] jsonBytes = Encoding.UTF8.GetBytes(json);
+
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<POCO>(json));
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<POCO>(jsonBytes));
+ Assert.Throws<JsonException>(() => JsonSerializer.DeserializeAsync<POCO>(new MemoryStream(jsonBytes)).Result);
+
+ // Using a reader directly throws since it can't read full object.
+ Utf8JsonReader reader = new Utf8JsonReader(jsonBytes);
+ try
+ {
+ JsonSerializer.Deserialize<POCO>(ref reader);
+ Assert.True(false, "Expected exception.");
+ }
+ catch (JsonException) { }
+
+ Assert.Equal(0, reader.BytesConsumed);
+ }
}
}
// See the LICENSE file in the project root for more information.
using System.Buffers;
+using System.IO;
+using System.Threading.Tasks;
using Xunit;
namespace System.Text.Json.Serialization.Tests
obj = JsonSerializer.Deserialize(ref reader, typeof(int));
Assert.Equal(1, obj);
}
+
+ [Theory]
+ [InlineData("0,1")]
+ [InlineData("0 1")]
+ [InlineData("0 {}")]
+ public static void TooMuchJson(string json)
+ {
+ byte[] jsonBytes = Encoding.UTF8.GetBytes(json);
+
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<int>(json));
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<int>(jsonBytes));
+ Assert.Throws<JsonException>(() => JsonSerializer.DeserializeAsync<int>(new MemoryStream(jsonBytes)).Result);
+
+ // Using a reader directly doesn't throw.
+ Utf8JsonReader reader = new Utf8JsonReader(jsonBytes);
+ JsonSerializer.Deserialize<int>(ref reader);
+ Assert.Equal(1, reader.BytesConsumed);
+ }
+
+ [Theory]
+ [InlineData("0/**/")]
+ [InlineData("0 /**/")]
+ public static void TooMuchJsonWithComments(string json)
+ {
+ byte[] jsonBytes = Encoding.UTF8.GetBytes(json);
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<int>(json));
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<int>(jsonBytes));
+ Assert.Throws<JsonException>(() => JsonSerializer.DeserializeAsync<int>(new MemoryStream(jsonBytes)).Result);
+
+ // Using a reader directly doesn't throw.
+ Utf8JsonReader reader = new Utf8JsonReader(jsonBytes);
+ JsonSerializer.Deserialize<int>(ref reader);
+ Assert.Equal(1, reader.BytesConsumed);
+
+ // Use JsonCommentHandling.Skip
+
+ var options = new JsonSerializerOptions();
+ options.ReadCommentHandling = JsonCommentHandling.Skip;
+ JsonSerializer.Deserialize<int>(json, options);
+ JsonSerializer.Deserialize<int>(jsonBytes, options);
+ int result = JsonSerializer.DeserializeAsync<int>(new MemoryStream(jsonBytes), options).Result;
+
+ // Using a reader directly doesn't throw.
+ reader = new Utf8JsonReader(jsonBytes);
+ JsonSerializer.Deserialize<int>(ref reader, options);
+ Assert.Equal(1, reader.BytesConsumed);
+ }
+
+ [Theory]
+ [InlineData("[")]
+ [InlineData("[0")]
+ [InlineData("[0,")]
+ public static void TooLittleJsonForIntArray(string json)
+ {
+ byte[] jsonBytes = Encoding.UTF8.GetBytes(json);
+
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<int[]>(json));
+ Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<int[]>(jsonBytes));
+ Assert.Throws<JsonException>(() => JsonSerializer.DeserializeAsync<int[]>(new MemoryStream(jsonBytes)).Result);
+
+ // Using a reader directly throws since it can't read full int[].
+ Utf8JsonReader reader = new Utf8JsonReader(jsonBytes);
+ try
+ {
+ JsonSerializer.Deserialize<int[]>(ref reader);
+ Assert.True(false, "Expected exception.");
+ }
+ catch (JsonException) { }
+
+ Assert.Equal(0, reader.BytesConsumed);
+ }
}
}
}
}
- // NOTE: LongInputString test is constrained to run on Windows and MacOSX because it causes
- // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can
- // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the
- // time the memory is accessed which triggers the full memory allocation.
+ private const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024;
private const int MaxExpansionFactorWhileTranscoding = 3;
- private const int MaxArrayLengthBeforeCalculatingSize = int.MaxValue / 2 / MaxExpansionFactorWhileTranscoding;
- [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
- [ConditionalTheory(nameof(IsX64))]
- [InlineData(MaxArrayLengthBeforeCalculatingSize - 3)]
- [InlineData(MaxArrayLengthBeforeCalculatingSize - 2)]
- [InlineData(MaxArrayLengthBeforeCalculatingSize - 1)]
- [InlineData(MaxArrayLengthBeforeCalculatingSize)]
- [InlineData(MaxArrayLengthBeforeCalculatingSize + 1)]
- [InlineData(MaxArrayLengthBeforeCalculatingSize + 2)]
- [InlineData(MaxArrayLengthBeforeCalculatingSize + 3)]
- [OuterLoop]
+ private const long Threshold = ArrayPoolMaxSizeBeforeUsingNormalAlloc / MaxExpansionFactorWhileTranscoding;
+
+ [Theory]
+ [InlineData(Threshold - 3)]
+ [InlineData(Threshold - 2)]
+ [InlineData(Threshold - 1)]
+ [InlineData(Threshold)]
+ [InlineData(Threshold + 1)]
+ [InlineData(Threshold + 2)]
+ [InlineData(Threshold + 3)]
public static void LongInputString(int length)
{
// Verify boundary conditions in Deserialize() that inspect the size to determine allocation strategy.
projects / platform / upstream / dotnet / runtime.git / commitdiff