From: Mandar Sahasrabuddhe Date: Tue, 21 May 2019 02:25:22 +0000 (+0530) Subject: Implement GetComment API (dotnet/corefx#35705) X-Git-Tag: submit/tizen/20210909.063632~11031^2~1530 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=3926f4411d47f0b2d1c2ca97a04529bb3dac42eb;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Implement GetComment API (dotnet/corefx#35705) * CoreFx dotnet/corefx#33347: Implement GetComment API * Generate References as per correct steps * Exclude delims & line separators, new unescape test * Revert consume comment changes, test case fixes * Partial revert of previous incorrect online merge * fixed another online merge conflict resolution slippage * Intermediate rebase from master * Reworked files * Reworked files 2 * Fix coding sytle as per review comments Commit migrated from https://github.com/dotnet/corefx/commit/53aaaf3c848e4e737420de2b46f7cace397d730a --- diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index 82fd50a..935efd7 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -197,6 +197,7 @@ namespace System.Text.Json public bool GetBoolean() { throw null; } public System.DateTime GetDateTime() { throw null; } public System.DateTimeOffset GetDateTimeOffset() { throw null; } + public string GetComment() { throw null; } public decimal GetDecimal() { throw null; } public double GetDouble() { throw null; } public System.Guid GetGuid() { throw null; } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs index b3460e1..201d6f9 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs @@ -17,7 +17,7 @@ namespace System.Text.Json /// Thrown if trying to get the value of the JSON token that is not a string /// (i.e. other than or ). /// - /// I will also throw when the JSON string contains invalid UTF-8 bytes, or invalid UTF-16 surrogates. + /// It will also throw when the JSON string contains invalid UTF-8 bytes, or invalid UTF-16 surrogates. /// public string GetString() { @@ -40,6 +40,23 @@ namespace System.Text.Json } /// + /// Reads the next JSON token value from the source as a comment, transcoded as a . + /// + /// + /// Thrown if trying to get the value of the JSON token that is not a comment. + /// + /// + public string GetComment() + { + if (TokenType != JsonTokenType.Comment) + { + throw ThrowHelper.GetInvalidOperationException_ExpectedComment(TokenType); + } + ReadOnlySpan span = HasValueSequence ? ValueSequence.ToArray() : ValueSpan; + return JsonReaderHelper.TranscodeHelper(span); + } + + /// /// Reads the next JSON token value from the source as a . /// Returns true if the TokenType is JsonTokenType.True and false if the TokenType is JsonTokenType.False. /// diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs index f044b51..ae373390c 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs @@ -187,6 +187,11 @@ namespace System.Text.Json return GetInvalidOperationException(tokenType); } + public static InvalidOperationException GetInvalidOperationException_ExpectedComment(JsonTokenType tokenType) + { + return GetInvalidOperationException("comment", tokenType); + } + [MethodImpl(MethodImplOptions.NoInlining)] private static InvalidOperationException GetInvalidOperationException(string message, JsonTokenType tokenType) { diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs index e91a260..642cb87 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Text.RegularExpressions; using Newtonsoft.Json; using Xunit; @@ -475,10 +476,11 @@ namespace System.Text.Json.Tests [Fact] public static void InvalidConversion() { - string jsonString = "[\"stringValue\", true, 1234]"; + string jsonString = "[\"stringValue\", true, /* Comment within */ 1234] // Comment outside"; byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); - var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state: default); + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = JsonCommentHandling.Allow }); + var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state); while (json.Read()) { if (json.TokenType != JsonTokenType.String) @@ -528,6 +530,17 @@ namespace System.Text.Json.Tests JsonTestHelper.AssertThrows(json, (jsonReader) => jsonReader.TryGetGuid(out _)); } + if (json.TokenType != JsonTokenType.Comment) + { + try + { + string value = json.GetComment(); + Assert.True(false, "Expected GetComment to throw InvalidOperationException due to mismatch token type."); + } + catch (InvalidOperationException) + { } + } + if (json.TokenType != JsonTokenType.True && json.TokenType != JsonTokenType.False) { try @@ -764,8 +777,6 @@ namespace System.Text.Json.Tests } } - - [Theory] [MemberData(nameof(InvalidUTF8Strings))] public static void TestingGetStringInvalidUTF8(byte[] dataUtf8) @@ -803,6 +814,53 @@ namespace System.Text.Json.Tests } [Theory] + [MemberData(nameof(GetCommentTestData))] + public static void TestingGetComment(string jsonData, string expected) + { + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonData); + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = JsonCommentHandling.Allow }); + var reader = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state); + + Assert.True(reader.Read()); + Assert.Equal(JsonTokenType.StartObject, reader.TokenType); + + Assert.True(reader.Read()); + Assert.Equal(JsonTokenType.Comment, reader.TokenType); + Assert.Equal(expected, reader.GetComment()); + + Assert.True(reader.Read()); + Assert.Equal(JsonTokenType.EndObject, reader.TokenType); + + Assert.False(reader.Read()); + } + + [Theory] + [MemberData(nameof(GetCommentUnescapeData))] + public static void TestGetCommentUnescape(string jsonData, string expected) + { + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonData); + var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = JsonCommentHandling.Allow }); + var reader = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state); + bool commentFound = false; + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonTokenType.Comment: + commentFound = true; + string comment = reader.GetComment(); + Assert.Equal(expected, comment); + Assert.NotEqual(Regex.Unescape(expected), comment); + break; + default: + Assert.True(false); + break; + } + } + Assert.True(commentFound); + } + + [Theory] [MemberData(nameof(JsonDateTimeTestData.ValidISO8601Tests), MemberType = typeof(JsonDateTimeTestData))] public static void TestingStringsConversionToDateTime(string jsonString, string expectedString) { diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs index 8f27fd6..7bb7a67 100644 --- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs +++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs @@ -1259,7 +1259,7 @@ namespace System.Text.Json.Tests foundComment = true; indexAfterFirstComment = json.BytesConsumed; Assert.Equal(indexAfterFirstComment, json.CurrentState.BytesConsumed); - string actualComment = Encoding.UTF8.GetString(json.ValueSpan.ToArray()); // TODO: https://github.com/dotnet/corefx/issues/33347 + string actualComment = json.GetComment(); Assert.Equal(expectedComment, actualComment); break; } @@ -1329,7 +1329,7 @@ namespace System.Text.Json.Tests foundComment = true; indexAfterFirstComment = json.BytesConsumed; Assert.Equal(indexAfterFirstComment, json.CurrentState.BytesConsumed); - string actualComment = Encoding.UTF8.GetString(json.ValueSpan.ToArray()); + string actualComment = json.GetComment(); Assert.Equal(expectedComment, actualComment); break; } @@ -1356,7 +1356,7 @@ namespace System.Text.Json.Tests foundComment = true; indexAfterFirstComment = jsonSlice.BytesConsumed; Assert.Equal(indexAfterFirstComment, jsonSlice.CurrentState.BytesConsumed); - string actualComment = Encoding.UTF8.GetString(jsonSlice.ValueSpan.ToArray()); + string actualComment = jsonSlice.GetComment(); Assert.Equal(expectedComment, actualComment); break; } @@ -1380,7 +1380,7 @@ namespace System.Text.Json.Tests foundComment = true; indexAfterFirstComment = jsonSlice.BytesConsumed; Assert.Equal(indexAfterFirstComment, jsonSlice.CurrentState.BytesConsumed); - string actualComment = Encoding.UTF8.GetString(jsonSlice.ValueSpan.ToArray()); + string actualComment = jsonSlice.GetComment(); Assert.Equal(expectedComment, actualComment); break; } @@ -2097,8 +2097,8 @@ namespace System.Text.Json.Tests case JsonTokenType.Comment: if (expected != null) { - byte[] data = json.HasValueSequence ? json.ValueSequence.ToArray() : json.ValueSpan.ToArray(); - Assert.Equal(expected, Encoding.UTF8.GetString(data)); + string actualComment = json.GetComment(); + Assert.Equal(expected, actualComment); } else { @@ -3333,5 +3333,57 @@ namespace System.Text.Json.Tests }; } } + + public static IEnumerable GetCommentTestData + { + get + { + var dataList = new List(); + foreach (string delim in new[] { "\r", "\r\n", "\n" }) + { + // NOTE: Leading and trailing spaces in the comments are significant. + var singleLineComment = " Single Line Comment "; + dataList.Add(new object[] { $"{{//{singleLineComment}{delim}}}", singleLineComment }); + + var multilineComment = $" Multiline {delim} Comment "; + dataList.Add(new object[] { $"{{/*{multilineComment}*/{delim}}}", multilineComment }); + } + return dataList; + } + } + + public static IEnumerable GetCommentUnescapeData + { + get + { + var dataList = new List(); + + var rawComments = new string[] + { + "A string with {0}valid UTF8 \\t tab", + "A string with {0}invalid UTF8 \\xc3\\x28", + "A string with {0}valid UTF16 \\u002e \\u0009 म", + "A string with {0}invalid UTF16 \\uDD1E" + }; + + // single line comments + foreach (string raw in rawComments) + { + string str = string.Format(raw, ""); + string cmt = "//" + str; + dataList.Add(new object[] { cmt, str }); + } + + // multiline comments + foreach (string raw in rawComments) + { + string str = string.Format(raw, "\n"); + string cmt = "/*" + str + "*/"; + dataList.Add(new object[] { cmt, str }); + } + + return dataList; + } + } } }