From fed1a6577c3d88b6a4db420cbdeb8f8f46bdb97b Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Mon, 8 Apr 2019 02:55:50 -0700 Subject: [PATCH] Add support for trailing commas within Utf8JsonReader (dotnet/corefx#36690) * Add support for trailing commas with single-segment tests. * Implement single-segment case and update tests. * Update multi-segment path and add tests. * Add more tests, add to state, and update rollback logic. Commit migrated from https://github.com/dotnet/corefx/commit/87fdc75beacc76e02c8e572ae3b0613b418200eb --- .../System.Text.Json/ref/System.Text.Json.cs | 1 + .../src/Resources/Strings.resx | 9 + .../Text/Json/Reader/JsonReaderOptions.cs | 7 + .../Text/Json/Reader/JsonReaderState.cs | 2 + .../Reader/Utf8JsonReader.MultiSegment.cs | 89 +++++ .../System/Text/Json/Reader/Utf8JsonReader.cs | 110 ++++++ .../src/System/Text/Json/ThrowHelper.cs | 14 +- .../tests/Utf8JsonReaderTests.MultiSegment.cs | 114 ++++++ .../tests/Utf8JsonReaderTests.cs | 338 ++++++++++++++++++ 9 files changed, 683 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index 3b93bd328d1..1b4c6cea083 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -119,6 +119,7 @@ namespace System.Text.Json private int _dummyPrimitive; public System.Text.Json.JsonCommentHandling CommentHandling { get { throw null; } set { } } public int MaxDepth { get { throw null; } set { } } + public bool AllowTrailingCommas { get { throw null; } set { } } } public partial struct JsonReaderState { diff --git a/src/libraries/System.Text.Json/src/Resources/Strings.resx b/src/libraries/System.Text.Json/src/Resources/Strings.resx index ed8a9fd881c..b83207b940c 100644 --- a/src/libraries/System.Text.Json/src/Resources/Strings.resx +++ b/src/libraries/System.Text.Json/src/Resources/Strings.resx @@ -312,4 +312,13 @@ The JSON value is not in a supported Guid format. + + '{0}' is an invalid start of a property name or value, after a comment. + + + The JSON array contains a trailing comma at the end which is not supported in this mode. Change the reader options. + + + The JSON object contains a trailing comma at the end which is not supported in this mode. Change the reader options. + \ No newline at end of file diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderOptions.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderOptions.cs index e3cf75eaee3..7ef94d8d751 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderOptions.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderOptions.cs @@ -34,5 +34,12 @@ namespace System.Text.Json _maxDepth = value; } } + + /// + /// Defines whether an extra comma at the end of a list of JSON values in an object or array + /// are allowed (and ignored) within the JSON payload being read. + /// By default, it's set to false, and the reader will throw a if it encounters a trailing comma. + /// + public bool AllowTrailingCommas { get; set; } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderState.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderState.cs index aee4920413c..be959058811 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderState.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderState.cs @@ -21,6 +21,7 @@ namespace System.Text.Json internal bool _isNotPrimitive; internal char _numberFormat; internal bool _stringHasEscaping; + internal bool _trailingCommaBeforeComment; internal JsonTokenType _tokenType; internal JsonTokenType _previousTokenType; internal JsonReaderOptions _readerOptions; @@ -64,6 +65,7 @@ namespace System.Text.Json _isNotPrimitive = default; _numberFormat = default; _stringHasEscaping = default; + _trailingCommaBeforeComment = default; _tokenType = default; _previousTokenType = default; _readerOptions = options; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs index 1997a9a006c..70563bb817f 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs @@ -36,6 +36,7 @@ namespace System.Text.Json _isNotPrimitive = state._isNotPrimitive; _numberFormat = state._numberFormat; _stringHasEscaping = state._stringHasEscaping; + _trailingCommaBeforeComment = state._trailingCommaBeforeComment; _tokenType = state._tokenType; _previousTokenType = state._previousTokenType; _readerOptions = state._readerOptions; @@ -373,6 +374,10 @@ namespace System.Text.Json { while (true) { + Debug.Assert((_trailingCommaBeforeComment && _readerOptions.CommentHandling == JsonCommentHandling.Allow) || !_trailingCommaBeforeComment); + Debug.Assert((_trailingCommaBeforeComment && marker != JsonConstants.Slash) || !_trailingCommaBeforeComment); + _trailingCommaBeforeComment = false; + if (marker == JsonConstants.Quote) { return ConsumeStringMultiSegment(); @@ -657,6 +662,8 @@ namespace System.Text.Json private bool ConsumePropertyNameMultiSegment() { + _trailingCommaBeforeComment = false; + if (!ConsumeStringMultiSegment()) { return false; @@ -1531,6 +1538,7 @@ namespace System.Text.Json long prevLineNumber = _lineNumber; JsonTokenType prevTokenType = _tokenType; SequencePosition prevSequencePosition = _currentPosition; + bool prevTrailingCommaBeforeComment = _trailingCommaBeforeComment; ConsumeTokenResult result = ConsumeNextTokenMultiSegment(marker); if (result == ConsumeTokenResult.Success) { @@ -1544,6 +1552,7 @@ namespace System.Text.Json _lineNumber = prevLineNumber; _totalConsumed = prevTotalConsumed; _currentPosition = prevSequencePosition; + _trailingCommaBeforeComment = prevTrailingCommaBeforeComment; } return false; } @@ -1619,6 +1628,7 @@ namespace System.Text.Json if (_readerOptions.CommentHandling == JsonCommentHandling.Allow && first == JsonConstants.Slash) { + _trailingCommaBeforeComment = true; return ConsumeCommentMultiSegment() ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } @@ -1626,12 +1636,30 @@ namespace System.Text.Json { if (first != JsonConstants.Quote) { + if (first == JsonConstants.CloseBrace) + { + if (_readerOptions.AllowTrailingCommas) + { + EndObject(); + return ConsumeTokenResult.Success; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd); + } ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyNotFound, first); } return ConsumePropertyNameMultiSegment() ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } else { + if (first == JsonConstants.CloseBracket) + { + if (_readerOptions.AllowTrailingCommas) + { + EndArray(); + return ConsumeTokenResult.Success; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd); + } return ConsumeValueMultiSegment(first) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } } @@ -1652,6 +1680,9 @@ namespace System.Text.Json private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentTokenMultiSegment() { + Debug.Assert(_readerOptions.CommentHandling == JsonCommentHandling.Allow); + Debug.Assert(_tokenType == JsonTokenType.Comment); + if (JsonReaderHelper.IsTokenTypePrimitive(_previousTokenType)) { _tokenType = _inObject ? JsonTokenType.StartObject : JsonTokenType.StartArray; @@ -1690,6 +1721,12 @@ namespace System.Text.Json if (first == JsonConstants.ListSeparator) { + // A comma without some JSON value preceding it is invalid + if (_previousTokenType <= JsonTokenType.StartObject || _previousTokenType == JsonTokenType.StartArray || _trailingCommaBeforeComment) + { + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyOrValueAfterComment, first); + } + _consumed++; _bytePositionInLine++; @@ -1726,10 +1763,33 @@ namespace System.Text.Json first = _buffer[_consumed]; } + if (first == JsonConstants.Slash) + { + _trailingCommaBeforeComment = true; + if (ConsumeCommentMultiSegment()) + { + goto Done; + } + else + { + goto RollBack; + } + } + if (_inObject) { if (first != JsonConstants.Quote) { + if (first == JsonConstants.CloseBrace) + { + if (_readerOptions.AllowTrailingCommas) + { + EndObject(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd); + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyNotFound, first); } if (ConsumePropertyNameMultiSegment()) @@ -1743,6 +1803,16 @@ namespace System.Text.Json } else { + if (first == JsonConstants.CloseBracket) + { + if (_readerOptions.AllowTrailingCommas) + { + EndArray(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd); + } + if (ConsumeValueMultiSegment(first)) { goto Done; @@ -2022,12 +2092,31 @@ namespace System.Text.Json { if (marker != JsonConstants.Quote) { + if (marker == JsonConstants.CloseBrace) + { + if (_readerOptions.AllowTrailingCommas) + { + EndObject(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd); + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyNotFound, marker); } return ConsumePropertyNameMultiSegment() ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } else { + if (marker == JsonConstants.CloseBracket) + { + if (_readerOptions.AllowTrailingCommas) + { + EndArray(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd); + } return ConsumeValueMultiSegment(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs index a7ea5263ea6..97bb44be3c4 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs @@ -41,6 +41,7 @@ namespace System.Text.Json private bool _isLastSegment; internal bool _stringHasEscaping; private readonly bool _isMultiSegment; + private bool _trailingCommaBeforeComment; private SequencePosition _nextPosition; private SequencePosition _currentPosition; @@ -154,6 +155,7 @@ namespace System.Text.Json _isNotPrimitive = _isNotPrimitive, _numberFormat = _numberFormat, _stringHasEscaping = _stringHasEscaping, + _trailingCommaBeforeComment = _trailingCommaBeforeComment, _tokenType = _tokenType, _previousTokenType = _previousTokenType, _readerOptions = _readerOptions, @@ -187,6 +189,7 @@ namespace System.Text.Json _isNotPrimitive = state._isNotPrimitive; _numberFormat = state._numberFormat; _stringHasEscaping = state._stringHasEscaping; + _trailingCommaBeforeComment = state._trailingCommaBeforeComment; _tokenType = state._tokenType; _previousTokenType = state._previousTokenType; _readerOptions = state._readerOptions; @@ -529,6 +532,15 @@ namespace System.Text.Json if (!_inObject || _bitStack.CurrentDepth <= 0) ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.MismatchedObjectArray, JsonConstants.CloseBrace); + if (_trailingCommaBeforeComment) + { + if (!_readerOptions.AllowTrailingCommas) + { + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd); + } + _trailingCommaBeforeComment = false; + } + _tokenType = JsonTokenType.EndObject; ValueSpan = _buffer.Slice(_consumed, 1); @@ -554,6 +566,15 @@ namespace System.Text.Json if (_inObject || _bitStack.CurrentDepth <= 0) ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.MismatchedObjectArray, JsonConstants.CloseBracket); + if (_trailingCommaBeforeComment) + { + if (!_readerOptions.AllowTrailingCommas) + { + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd); + } + _trailingCommaBeforeComment = false; + } + _tokenType = JsonTokenType.EndArray; ValueSpan = _buffer.Slice(_consumed, 1); @@ -798,6 +819,10 @@ namespace System.Text.Json { while (true) { + Debug.Assert((_trailingCommaBeforeComment && _readerOptions.CommentHandling == JsonCommentHandling.Allow) || !_trailingCommaBeforeComment); + Debug.Assert((_trailingCommaBeforeComment && marker != JsonConstants.Slash) || !_trailingCommaBeforeComment); + _trailingCommaBeforeComment = false; + if (marker == JsonConstants.Quote) { return ConsumeString(); @@ -978,6 +1003,8 @@ namespace System.Text.Json private bool ConsumePropertyName() { + _trailingCommaBeforeComment = false; + if (!ConsumeString()) { return false; @@ -1449,6 +1476,7 @@ namespace System.Text.Json long prevPosition = _bytePositionInLine; long prevLineNumber = _lineNumber; JsonTokenType prevTokenType = _tokenType; + bool prevTrailingCommaBeforeComment = _trailingCommaBeforeComment; ConsumeTokenResult result = ConsumeNextToken(marker); if (result == ConsumeTokenResult.Success) { @@ -1460,6 +1488,7 @@ namespace System.Text.Json _tokenType = prevTokenType; _bytePositionInLine = prevPosition; _lineNumber = prevLineNumber; + _trailingCommaBeforeComment = prevTrailingCommaBeforeComment; } return false; } @@ -1526,6 +1555,7 @@ namespace System.Text.Json if (_readerOptions.CommentHandling == JsonCommentHandling.Allow && first == JsonConstants.Slash) { + _trailingCommaBeforeComment = true; return ConsumeComment() ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } @@ -1533,12 +1563,30 @@ namespace System.Text.Json { if (first != JsonConstants.Quote) { + if (first == JsonConstants.CloseBrace) + { + if (_readerOptions.AllowTrailingCommas) + { + EndObject(); + return ConsumeTokenResult.Success; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd); + } ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyNotFound, first); } return ConsumePropertyName() ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } else { + if (first == JsonConstants.CloseBracket) + { + if (_readerOptions.AllowTrailingCommas) + { + EndArray(); + return ConsumeTokenResult.Success; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd); + } return ConsumeValue(first) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } } @@ -1559,6 +1607,9 @@ namespace System.Text.Json private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentToken() { + Debug.Assert(_readerOptions.CommentHandling == JsonCommentHandling.Allow); + Debug.Assert(_tokenType == JsonTokenType.Comment); + if (JsonReaderHelper.IsTokenTypePrimitive(_previousTokenType)) { _tokenType = _inObject ? JsonTokenType.StartObject : JsonTokenType.StartArray; @@ -1597,6 +1648,12 @@ namespace System.Text.Json if (first == JsonConstants.ListSeparator) { + // A comma without some JSON value preceding it is invalid + if (_previousTokenType <= JsonTokenType.StartObject || _previousTokenType == JsonTokenType.StartArray || _trailingCommaBeforeComment) + { + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyOrValueAfterComment, first); + } + _consumed++; _bytePositionInLine++; @@ -1624,10 +1681,33 @@ namespace System.Text.Json first = _buffer[_consumed]; } + if (first == JsonConstants.Slash) + { + _trailingCommaBeforeComment = true; + if (ConsumeComment()) + { + goto Done; + } + else + { + goto RollBack; + } + } + if (_inObject) { if (first != JsonConstants.Quote) { + if (first == JsonConstants.CloseBrace) + { + if (_readerOptions.AllowTrailingCommas) + { + EndObject(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd); + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyNotFound, first); } if (ConsumePropertyName()) @@ -1641,6 +1721,16 @@ namespace System.Text.Json } else { + if (first == JsonConstants.CloseBracket) + { + if (_readerOptions.AllowTrailingCommas) + { + EndArray(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd); + } + if (ConsumeValue(first)) { goto Done; @@ -1905,12 +1995,32 @@ namespace System.Text.Json { if (marker != JsonConstants.Quote) { + if (marker == JsonConstants.CloseBrace) + { + if (_readerOptions.AllowTrailingCommas) + { + EndObject(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd); + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedStartOfPropertyNotFound, marker); } return ConsumePropertyName() ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } else { + if (marker == JsonConstants.CloseBracket) + { + if (_readerOptions.AllowTrailingCommas) + { + EndArray(); + goto Done; + } + ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd); + } + return ConsumeValue(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs index 3ba9fec9e84..cdbd7a4668b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs @@ -258,6 +258,12 @@ namespace System.Text.Json case ExceptionResource.MismatchedObjectArray: message = SR.Format(SR.MismatchedObjectArray, character); break; + case ExceptionResource.TrailingCommaNotAllowedBeforeArrayEnd: + message = SR.TrailingCommaNotAllowedBeforeArrayEnd; + break; + case ExceptionResource.TrailingCommaNotAllowedBeforeObjectEnd: + message = SR.TrailingCommaNotAllowedBeforeObjectEnd; + break; case ExceptionResource.EndOfStringNotFound: message = SR.EndOfStringNotFound; break; @@ -288,6 +294,9 @@ namespace System.Text.Json case ExceptionResource.ExpectedStartOfPropertyOrValueNotFound: message = SR.ExpectedStartOfPropertyOrValueNotFound; break; + case ExceptionResource.ExpectedStartOfPropertyOrValueAfterComment: + message = SR.Format(SR.ExpectedStartOfPropertyOrValueAfterComment, character); + break; case ExceptionResource.ExpectedStartOfValueNotFound: message = SR.Format(SR.ExpectedStartOfValueNotFound, character); break; @@ -506,6 +515,7 @@ namespace System.Text.Json ExpectedSeparatorAfterPropertyNameNotFound, ExpectedStartOfPropertyNotFound, ExpectedStartOfPropertyOrValueNotFound, + ExpectedStartOfPropertyOrValueAfterComment, ExpectedStartOfValueNotFound, ExpectedTrue, ExpectedValueAfterPropertyNameNotFound, @@ -525,7 +535,9 @@ namespace System.Text.Json FailedToGetMinimumSizeSpan, FailedToGetLargerSpan, CannotWritePropertyWithinArray, - ExpectedJsonTokens + ExpectedJsonTokens, + TrailingCommaNotAllowedBeforeArrayEnd, + TrailingCommaNotAllowedBeforeObjectEnd, } internal enum NumericType diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs index 7d3a6e07956..965c863561c 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs @@ -744,5 +744,119 @@ namespace System.Text.Json.Tests } }); } + + [Theory] + [MemberData(nameof(JsonWithValidTrailingCommas))] + public static void JsonWithTrailingCommasMultiSegment_Valid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + ReadOnlySequence sequence = JsonTestHelper.GetSequence(utf8, 1); + + { + JsonReaderState state = default; + TrailingCommasHelper(sequence, state, allow: false, expectThrow: true); + } + + { + var state = new JsonReaderState(options: default); + TrailingCommasHelper(sequence, state, allow: false, expectThrow: true); + } + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(sequence, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(sequence, state, allowTrailingCommas, expectThrow: false); + } + } + + [Theory] + [MemberData(nameof(JsonWithInvalidTrailingCommas))] + public static void JsonWithTrailingCommasMultiSegment_Invalid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + ReadOnlySequence sequence = JsonTestHelper.GetSequence(utf8, 1); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(sequence, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(sequence, state, allowTrailingCommas, expectThrow: true); + } + } + + [Theory] + [MemberData(nameof(JsonWithValidTrailingCommasAndComments))] + public static void JsonWithTrailingCommasAndCommentsMultiSegment_Valid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + ReadOnlySequence sequence = JsonTestHelper.GetSequence(utf8, 1); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + if (commentHandling == JsonCommentHandling.Disallow) + { + continue; + } + + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(sequence, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(sequence, state, allowTrailingCommas, expectThrow: false); + } + } + + [Theory] + [MemberData(nameof(JsonWithInvalidTrailingCommasAndComments))] + public static void JsonWithTrailingCommasAndCommentsMultiSegment_Invalid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + ReadOnlySequence sequence = JsonTestHelper.GetSequence(utf8, 1); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + if (commentHandling == JsonCommentHandling.Disallow) + { + continue; + } + + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(sequence, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(sequence, state, allowTrailingCommas, expectThrow: true); + } + } + + private static void TrailingCommasHelper(ReadOnlySequence utf8, JsonReaderState state, bool allow, bool expectThrow) + { + var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state); + + Assert.Equal(allow, state.Options.AllowTrailingCommas); + Assert.Equal(allow, reader.CurrentState.Options.AllowTrailingCommas); + + if (expectThrow) + { + JsonTestHelper.AssertThrows(reader, (jsonReader) => + { + while (jsonReader.Read()) + ; + }); + } + else + { + while (reader.Read()) + ; + } + } } } diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs index 831b45c98ae..f7987db6c45 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs @@ -30,6 +30,7 @@ namespace System.Text.Json.Tests Assert.Equal(0, json.CurrentState.BytesConsumed); Assert.Equal(default, json.CurrentState.Position); Assert.Equal(0, json.CurrentState.Options.MaxDepth); + Assert.False(json.CurrentState.Options.AllowTrailingCommas); Assert.Equal(JsonCommentHandling.Disallow, json.CurrentState.Options.CommentHandling); Assert.False(json.Read()); @@ -2120,6 +2121,233 @@ namespace System.Text.Json.Tests }); } + [Theory] + [MemberData(nameof(JsonWithValidTrailingCommas))] + public static void JsonWithTrailingCommas_Valid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + { + JsonReaderState state = default; + TrailingCommasHelper(utf8, state, allow: false, expectThrow: true); + } + + { + var state = new JsonReaderState(options: default); + TrailingCommasHelper(utf8, state, allow: false, expectThrow: true); + } + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(utf8, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(utf8, state, allowTrailingCommas, expectThrow: false); + } + } + + [Theory] + [MemberData(nameof(JsonWithInvalidTrailingCommas))] + public static void JsonWithTrailingCommas_Invalid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(utf8, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(utf8, state, allowTrailingCommas, expectThrow: true); + } + } + + [Theory] + [MemberData(nameof(JsonWithValidTrailingCommasAndComments))] + public static void JsonWithTrailingCommasAndComments_Valid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + if (commentHandling == JsonCommentHandling.Disallow) + { + continue; + } + + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(utf8, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(utf8, state, allowTrailingCommas, expectThrow: false); + } + } + + [Theory] + [MemberData(nameof(JsonWithInvalidTrailingCommasAndComments))] + public static void JsonWithTrailingCommasAndComments_Invalid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + if (commentHandling == JsonCommentHandling.Disallow) + { + continue; + } + + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling }); + TrailingCommasHelper(utf8, state, allow: false, expectThrow: true); + + bool allowTrailingCommas = true; + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = allowTrailingCommas }); + TrailingCommasHelper(utf8, state, allowTrailingCommas, expectThrow: true); + } + } + + private static void TrailingCommasHelper(byte[] utf8, JsonReaderState state, bool allow, bool expectThrow) + { + var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state); + + Assert.Equal(allow, state.Options.AllowTrailingCommas); + Assert.Equal(allow, reader.CurrentState.Options.AllowTrailingCommas); + + if (expectThrow) + { + JsonTestHelper.AssertThrows(reader, (jsonReader) => + { + while (jsonReader.Read()) + ; + }); + } + else + { + while (reader.Read()) + ; + } + } + + [Theory] + [MemberData(nameof(JsonWithValidTrailingCommas))] + public static void PartialJsonWithTrailingCommas_Valid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + { + JsonReaderState state = default; + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + } + + { + var state = new JsonReaderState(options: default); + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + } + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = false }); + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = true }); + TrailingCommasHelperPartial(utf8, state, expectThrow: false); + } + } + + [Theory] + [MemberData(nameof(JsonWithValidTrailingCommasAndComments))] + public static void PartialJsonWithTrailingCommasAndComments_Valid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + if (commentHandling == JsonCommentHandling.Disallow) + { + continue; + } + + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = false }); + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = true }); + TrailingCommasHelperPartial(utf8, state, expectThrow: false); + } + } + + [Theory] + [MemberData(nameof(JsonWithInvalidTrailingCommas))] + public static void PartialJsonWithTrailingCommas_Invalid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = false }); + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = true }); + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + } + } + + [Theory] + [MemberData(nameof(JsonWithInvalidTrailingCommasAndComments))] + public static void PartialJsonWithTrailingCommasAndComments_Invalid(string jsonString) + { + byte[] utf8 = Encoding.UTF8.GetBytes(jsonString); + + foreach (JsonCommentHandling commentHandling in Enum.GetValues(typeof(JsonCommentHandling))) + { + if (commentHandling == JsonCommentHandling.Disallow) + { + continue; + } + + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = false }); + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + + state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = commentHandling, AllowTrailingCommas = true }); + TrailingCommasHelperPartial(utf8, state, expectThrow: true); + } + } + + private static void TrailingCommasHelperPartial(byte[] utf8, JsonReaderState state, bool expectThrow) + { + if (expectThrow) + { + Assert.Throws(() => PartialReaderLoop(utf8, state)); + } + else + { + PartialReaderLoop(utf8, state); + } + } + + private static void PartialReaderLoop(byte[] utf8, JsonReaderState state) + { + for (int i = 0; i < utf8.Length; i++) + { + JsonReaderState stateCopy = state; + PartialReaderLoop(utf8, stateCopy, i); + } + } + + private static void PartialReaderLoop(byte[] utf8, JsonReaderState state, int splitLocation) + { + var reader = new Utf8JsonReader(utf8.AsSpan(0, splitLocation), isFinalBlock: false, state); + while (reader.Read()) + ; + + long consumed = reader.BytesConsumed; + reader = new Utf8JsonReader(utf8.AsSpan((int)consumed), isFinalBlock: true, reader.CurrentState); + while (reader.Read()) + ; + } + public static IEnumerable TestCases { get @@ -2345,6 +2573,116 @@ namespace System.Text.Json.Tests } } + public static IEnumerable JsonWithValidTrailingCommas + { + get + { + return new List + { + new object[] {"{\"name\": \"value\",}"}, + new object[] {"{\"name\": [],}"}, + new object[] {"{\"name\": 1,}"}, + new object[] {"{\"name\": true,}"}, + new object[] {"{\"name\": false,}"}, + new object[] {"{\"name\": null,}"}, + new object[] {"{\"name\": [{},],}"}, + new object[] {"{\"first\" : \"value\", \"name\": [{},], \"last\":2 ,}"}, + new object[] {"{\"prop\":{\"name\": 1,\"last\":2,},}"}, + new object[] {"{\"prop\":[1,2,],}"}, + new object[] {"[\"value\",]"}, + new object[] {"[1,]"}, + new object[] {"[true,]"}, + new object[] {"[false,]"}, + new object[] {"[null,]"}, + new object[] {"[{},]"}, + new object[] {"[{\"name\": [],},]"}, + new object[] {"[1, {\"name\": [],},2 , ]"}, + new object[] {"[[1,2,],]"}, + new object[] {"[{\"name\": 1,\"last\":2,},]"}, + }; + } + } + + public static IEnumerable JsonWithValidTrailingCommasAndComments + { + get + { + return new List + { + new object[] {"{\"name\": \"value\"/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": []/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": 1/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": true/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": false/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": null/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": [{},]/*comment*/,/*comment*/}"}, + new object[] {"{\"first\" : \"value\", \"name\": [{},], \"last\":2 /*comment*/,/*comment*/}"}, + new object[] {"{\"prop\":{\"name\": 1,\"last\":2,}/*comment*/,}"}, + new object[] {"{\"prop\":[1,2,]/*comment*/,}"}, + new object[] {"{\"prop\":1,/*comment*/}"}, + new object[] {"[\"value\"/*comment*/,/*comment*/]"}, + new object[] {"[1/*comment*/,/*comment*/]"}, + new object[] {"[true/*comment*/,/*comment*/]"}, + new object[] {"[false/*comment*/,/*comment*/]"}, + new object[] {"[null/*comment*/,/*comment*/]"}, + new object[] {"[{}/*comment*/,/*comment*/]"}, + new object[] {"[{\"name\": [],}/*comment*/,/*comment*/]"}, + new object[] {"[1, {\"name\": [],},2 /*comment*/,/*comment*/ ]"}, + new object[] {"[[1,2,]/*comment*/,]"}, + new object[] {"[{\"name\": 1,\"last\":2,}/*comment*/,]"}, + new object[] {"[1,/*comment*/]"}, + }; + } + } + + public static IEnumerable JsonWithInvalidTrailingCommas + { + get + { + return new List + { + new object[] {","}, + new object[] {" , "}, + new object[] {"{},"}, + new object[] {"[],"}, + new object[] {"1,"}, + new object[] {"true,"}, + new object[] {"false,"}, + new object[] {"null,"}, + new object[] {"{,}"}, + new object[] {"{\"name\": 1,,}"}, + new object[] {"{\"name\": 1,,\"last\":2,}"}, + new object[] {"[,]"}, + new object[] {"[1,,]"}, + new object[] {"[1,,2,]"}, + }; + } + } + + public static IEnumerable JsonWithInvalidTrailingCommasAndComments + { + get + { + return new List + { + new object[] {"/*comment*/ ,/*comment*/"}, + new object[] {" /*comment*/ , /*comment*/ "}, + new object[] {"{}/*comment*/,/*comment*/"}, + new object[] {"[]/*comment*/,/*comment*/"}, + new object[] {"1/*comment*/,/*comment*/"}, + new object[] {"true/*comment*/,/*comment*/"}, + new object[] {"false/*comment*/,/*comment*/"}, + new object[] {"null/*comment*/,/*comment*/"}, + new object[] {"{/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": 1/*comment*/,/*comment*/,/*comment*/}"}, + new object[] {"{\"name\": 1,/*comment*/,\"last\":2,}"}, + new object[] {"[/*comment*/,/*comment*/]"}, + new object[] {"[1/*comment*/,/*comment*/,/*comment*/]"}, + new object[] {"[1,/*comment*/,2,]"}, + }; + } + } + public static IEnumerable InvalidJsonStrings { get -- 2.34.1