Implement GetComment API (dotnet/corefx#35705)

* CoreFx dotnet/corefx#33347: Implement GetComment API

* Generate References as per correct steps

* Exclude delims & line separators, new unescape test

* Revert consume comment changes, test case fixes

* Partial revert of previous incorrect online merge

* fixed another online merge conflict resolution slippage

* Intermediate rebase from master

* Reworked files

* Reworked files 2

* Fix coding sytle as per review comments

Commit migrated from https://github.com/dotnet/corefx/commit/53aaaf3c848e4e737420de2b46f7cace397d730a

diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs
index 82fd50a..935efd7 100644 (file)
--- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs
+++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs
@@ -197,6 +197,7 @@ namespace System.Text.Json
         public bool GetBoolean() { throw null; }
         public System.DateTime GetDateTime() { throw null; }
         public System.DateTimeOffset GetDateTimeOffset() { throw null; }
+        public string GetComment() { throw null; }
         public decimal GetDecimal() { throw null; }
         public double GetDouble() { throw null; }
         public System.Guid GetGuid() { throw null; }

diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs
index b3460e1..201d6f9 100644 (file)
--- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs
+++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.TryGet.cs
@@ -17,7 +17,7 @@ namespace System.Text.Json
         /// Thrown if trying to get the value of the JSON token that is not a string
         /// (i.e. other than <see cref="JsonTokenType.String"/> or <see cref="JsonTokenType.PropertyName"/>).
         /// <seealso cref="TokenType" />
-        /// I will also throw when the JSON string contains invalid UTF-8 bytes, or invalid UTF-16 surrogates.
+        /// It will also throw when the JSON string contains invalid UTF-8 bytes, or invalid UTF-16 surrogates.
         /// </exception>
         public string GetString()
         {
@@ -40,6 +40,23 @@ namespace System.Text.Json
         }
 
         /// <summary>
+        /// Reads the next JSON token value from the source as a comment, transcoded as a <see cref="string"/>.
+        /// </summary>
+        /// <exception cref="InvalidOperationException">
+        /// Thrown if trying to get the value of the JSON token that is not a comment.
+        /// <seealso cref="TokenType" />
+        /// </exception>
+        public string GetComment()
+        {
+            if (TokenType != JsonTokenType.Comment)
+            {
+                throw ThrowHelper.GetInvalidOperationException_ExpectedComment(TokenType);
+            }
+            ReadOnlySpan<byte> span = HasValueSequence ? ValueSequence.ToArray() : ValueSpan;
+            return JsonReaderHelper.TranscodeHelper(span);
+        }
+
+        /// <summary>
         /// Reads the next JSON token value from the source as a <see cref="bool"/>.
         /// Returns true if the TokenType is JsonTokenType.True and false if the TokenType is JsonTokenType.False.
         /// </summary>

diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs
index f044b51..ae37339 100644 (file)
--- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs
+++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs
@@ -187,6 +187,11 @@ namespace System.Text.Json
             return GetInvalidOperationException(tokenType);
         }
 
+        public static InvalidOperationException GetInvalidOperationException_ExpectedComment(JsonTokenType tokenType)
+        {
+            return GetInvalidOperationException("comment", tokenType);
+        }
+
         [MethodImpl(MethodImplOptions.NoInlining)]
         private static InvalidOperationException GetInvalidOperationException(string message, JsonTokenType tokenType)
         {

diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs
index e91a260..642cb87 100644 (file)
--- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs
+++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.TryGet.cs
@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Text.RegularExpressions;
 using Newtonsoft.Json;
 using Xunit;
 
@@ -475,10 +476,11 @@ namespace System.Text.Json.Tests
         [Fact]
         public static void InvalidConversion()
         {
-            string jsonString = "[\"stringValue\", true, 1234]";
+            string jsonString = "[\"stringValue\", true, /* Comment within */ 1234] // Comment outside";
             byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
 
-            var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state: default);
+            var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = JsonCommentHandling.Allow });
+            var json = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state);
             while (json.Read())
             {
                 if (json.TokenType != JsonTokenType.String)
@@ -528,6 +530,17 @@ namespace System.Text.Json.Tests
                     JsonTestHelper.AssertThrows<InvalidOperationException>(json, (jsonReader) => jsonReader.TryGetGuid(out _));
                 }
 
+                if (json.TokenType != JsonTokenType.Comment)
+                {
+                    try
+                    {
+                        string value = json.GetComment();
+                        Assert.True(false, "Expected GetComment to throw InvalidOperationException due to mismatch token type.");
+                    }
+                    catch (InvalidOperationException)
+                    { }
+                }
+
                 if (json.TokenType != JsonTokenType.True && json.TokenType != JsonTokenType.False)
                 {
                     try
@@ -764,8 +777,6 @@ namespace System.Text.Json.Tests
             }
         }
 
-
-
         [Theory]
         [MemberData(nameof(InvalidUTF8Strings))]
         public static void TestingGetStringInvalidUTF8(byte[] dataUtf8)
@@ -803,6 +814,53 @@ namespace System.Text.Json.Tests
         }
 
         [Theory]
+        [MemberData(nameof(GetCommentTestData))]
+        public static void TestingGetComment(string jsonData, string expected)
+        {
+            byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonData);
+            var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = JsonCommentHandling.Allow });
+            var reader = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state);
+
+            Assert.True(reader.Read());
+            Assert.Equal(JsonTokenType.StartObject, reader.TokenType);
+
+            Assert.True(reader.Read());
+            Assert.Equal(JsonTokenType.Comment, reader.TokenType);
+            Assert.Equal(expected, reader.GetComment());
+
+            Assert.True(reader.Read());
+            Assert.Equal(JsonTokenType.EndObject, reader.TokenType);
+
+            Assert.False(reader.Read());
+        }
+
+        [Theory]
+        [MemberData(nameof(GetCommentUnescapeData))]
+        public static void TestGetCommentUnescape(string jsonData, string expected)
+        {
+            byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonData);
+            var state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = JsonCommentHandling.Allow });
+            var reader = new Utf8JsonReader(dataUtf8, isFinalBlock: true, state);
+            bool commentFound = false;
+            while (reader.Read())
+            {
+                switch (reader.TokenType)
+                {
+                    case JsonTokenType.Comment:
+                        commentFound = true;
+                        string comment = reader.GetComment();
+                        Assert.Equal(expected, comment);
+                        Assert.NotEqual(Regex.Unescape(expected), comment);
+                        break;
+                    default:
+                        Assert.True(false);
+                        break;
+                }
+            }
+            Assert.True(commentFound);
+        }
+
+        [Theory]
         [MemberData(nameof(JsonDateTimeTestData.ValidISO8601Tests), MemberType = typeof(JsonDateTimeTestData))]
         public static void TestingStringsConversionToDateTime(string jsonString, string expectedString)
         {

diff --git a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs
index 8f27fd6..7bb7a67 100644 (file)
--- a/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs
+++ b/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.cs
@@ -1259,7 +1259,7 @@ namespace System.Text.Json.Tests
                         foundComment = true;
                         indexAfterFirstComment = json.BytesConsumed;
                         Assert.Equal(indexAfterFirstComment, json.CurrentState.BytesConsumed);
-                        string actualComment = Encoding.UTF8.GetString(json.ValueSpan.ToArray()); // TODO: https://github.com/dotnet/corefx/issues/33347
+                        string actualComment = json.GetComment();
                         Assert.Equal(expectedComment, actualComment);
                         break;
                 }
@@ -1329,7 +1329,7 @@ namespace System.Text.Json.Tests
                         foundComment = true;
                         indexAfterFirstComment = json.BytesConsumed;
                         Assert.Equal(indexAfterFirstComment, json.CurrentState.BytesConsumed);
-                        string actualComment = Encoding.UTF8.GetString(json.ValueSpan.ToArray());
+                        string actualComment = json.GetComment();
                         Assert.Equal(expectedComment, actualComment);
                         break;
                 }
@@ -1356,7 +1356,7 @@ namespace System.Text.Json.Tests
                             foundComment = true;
                             indexAfterFirstComment = jsonSlice.BytesConsumed;
                             Assert.Equal(indexAfterFirstComment, jsonSlice.CurrentState.BytesConsumed);
-                            string actualComment = Encoding.UTF8.GetString(jsonSlice.ValueSpan.ToArray());
+                            string actualComment = jsonSlice.GetComment();
                             Assert.Equal(expectedComment, actualComment);
                             break;
                     }
@@ -1380,7 +1380,7 @@ namespace System.Text.Json.Tests
                                 foundComment = true;
                                 indexAfterFirstComment = jsonSlice.BytesConsumed;
                                 Assert.Equal(indexAfterFirstComment, jsonSlice.CurrentState.BytesConsumed);
-                                string actualComment = Encoding.UTF8.GetString(jsonSlice.ValueSpan.ToArray());
+                                string actualComment = jsonSlice.GetComment();
                                 Assert.Equal(expectedComment, actualComment);
                                 break;
                         }
@@ -2097,8 +2097,8 @@ namespace System.Text.Json.Tests
                     case JsonTokenType.Comment:
                         if (expected != null)
                         {
-                            byte[] data = json.HasValueSequence ? json.ValueSequence.ToArray() : json.ValueSpan.ToArray();
-                            Assert.Equal(expected, Encoding.UTF8.GetString(data));
+                            string actualComment = json.GetComment();
+                            Assert.Equal(expected, actualComment);
                         }
                         else
                         {
@@ -3333,5 +3333,57 @@ namespace System.Text.Json.Tests
                 };
             }
         }
+
+        public static IEnumerable<object[]> GetCommentTestData
+        {
+            get
+            {
+                var dataList = new List<object[]>();
+                foreach (string delim in new[] { "\r", "\r\n", "\n" })
+                {
+                    // NOTE: Leading and trailing spaces in the comments are significant.
+                    var singleLineComment = " Single Line Comment ";
+                    dataList.Add(new object[] { $"{{//{singleLineComment}{delim}}}", singleLineComment });
+
+                    var multilineComment = $" Multiline {delim} Comment ";
+                    dataList.Add(new object[] { $"{{/*{multilineComment}*/{delim}}}", multilineComment });
+                }
+                return dataList;
+            }
+        }
+
+        public static IEnumerable<object[]> GetCommentUnescapeData
+        {
+            get
+            {
+                var dataList = new List<object[]>();
+
+                var rawComments = new string[]
+                {
+                    "A string with {0}valid UTF8 \\t tab",
+                    "A string with {0}invalid UTF8 \\xc3\\x28",
+                    "A string with {0}valid UTF16 \\u002e \\u0009 म",
+                    "A string with {0}invalid UTF16 \\uDD1E"
+                };
+
+                // single line comments
+                foreach (string raw in rawComments)
+                {
+                    string str = string.Format(raw, "");
+                    string cmt = "//" + str;
+                    dataList.Add(new object[] { cmt, str });
+                }
+
+                // multiline comments
+                foreach (string raw in rawComments)
+                {
+                    string str = string.Format(raw, "\n");
+                    string cmt = "/*" + str + "*/";
+                    dataList.Add(new object[] { cmt, str });
+                }
+
+                return dataList;
+            }
+        }
     }
 }