Fix Regex handling of control characters (#68944)

It was using the C category rather than Cc category.  And the test for this was missing the combined categories.

diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs
index ef42be8..e3744fc 100644 (file)
--- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs
+++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs
@@ -54,8 +54,8 @@ namespace System.Text.RegularExpressions
         internal const string NotWordClass = "\u0000\u0000\u000A\u0000\uFFFE\uFFFC\uFFFB\uFFFD\uFFFF\uFFFA\uFFF7\uFFED\u0000"; // \W
         internal const string DigitClass = "\u0000\u0000\u0001\u0009"; // \d
         internal const string NotDigitClass = "\u0000\u0000\u0001\uFFF7"; // \D
-        internal const string ControlClass = "\0\0\a\0\u000f\u0010\u001e\u0012\u0011\0"; // \p{C}
-        internal const string NotControlClass = "\0\0\u0007\0\ufff1\ufff0\uffe2\uffee\uffef\0"; // \P{C}
+        internal const string ControlClass = "\0\0\u0001\u000f"; // \p{Cc}
+        internal const string NotControlClass = "\0\0\u0001\ufff1"; // \P{Cc}
         internal const string LetterClass = "\0\0\a\0\u0002\u0004\u0005\u0003\u0001\0"; // \p{L}
         internal const string NotLetterClass = "\0\0\u0007\0\ufffe\ufffc\ufffb\ufffd\uffff\0"; // \P{L}
         internal const string LetterOrDigitClass = "\0\0\b\0\u0002\u0004\u0005\u0003\u0001\0\t"; // [\p{L}\d]

diff --git a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexCharacterSetTests.cs b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexCharacterSetTests.cs
index 9db13f6..e3b1c95 100644 (file)
--- a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexCharacterSetTests.cs
+++ b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexCharacterSetTests.cs
@@ -359,53 +359,77 @@ namespace System.Text.RegularExpressions.Tests
         {
             foreach (RegexEngine engine in RegexHelpers.AvailableEngines)
             {
-                yield return new object[] { engine, "Cc", UnicodeCategory.Control };
-                yield return new object[] { engine, "Cf", UnicodeCategory.Format };
-                yield return new object[] { engine, "Cn", UnicodeCategory.OtherNotAssigned };
-                yield return new object[] { engine, "Co", UnicodeCategory.PrivateUse };
-                yield return new object[] { engine, "Cs", UnicodeCategory.Surrogate };
-                yield return new object[] { engine, "Ll", UnicodeCategory.LowercaseLetter };
-                yield return new object[] { engine, "Lm", UnicodeCategory.ModifierLetter };
-                yield return new object[] { engine, "Lo", UnicodeCategory.OtherLetter };
-                yield return new object[] { engine, "Lt", UnicodeCategory.TitlecaseLetter };
-                yield return new object[] { engine, "Lu", UnicodeCategory.UppercaseLetter };
-                yield return new object[] { engine, "Mc", UnicodeCategory.SpacingCombiningMark };
-                yield return new object[] { engine, "Me", UnicodeCategory.EnclosingMark };
-                yield return new object[] { engine, "Mn", UnicodeCategory.NonSpacingMark };
-                yield return new object[] { engine, "Nd", UnicodeCategory.DecimalDigitNumber };
-                yield return new object[] { engine, "Nl", UnicodeCategory.LetterNumber };
-                yield return new object[] { engine, "No", UnicodeCategory.OtherNumber };
-                yield return new object[] { engine, "Pc", UnicodeCategory.ConnectorPunctuation };
-                yield return new object[] { engine, "Pd", UnicodeCategory.DashPunctuation };
-                yield return new object[] { engine, "Pe", UnicodeCategory.ClosePunctuation };
-                yield return new object[] { engine, "Po", UnicodeCategory.OtherPunctuation };
-                yield return new object[] { engine, "Ps", UnicodeCategory.OpenPunctuation };
-                yield return new object[] { engine, "Pf", UnicodeCategory.FinalQuotePunctuation };
-                yield return new object[] { engine, "Pi", UnicodeCategory.InitialQuotePunctuation };
-                yield return new object[] { engine, "Sc", UnicodeCategory.CurrencySymbol };
-                yield return new object[] { engine, "Sk", UnicodeCategory.ModifierSymbol };
-                yield return new object[] { engine, "Sm", UnicodeCategory.MathSymbol };
-                yield return new object[] { engine, "So", UnicodeCategory.OtherSymbol };
-                yield return new object[] { engine, "Zl", UnicodeCategory.LineSeparator };
-                yield return new object[] { engine, "Zp", UnicodeCategory.ParagraphSeparator };
-                yield return new object[] { engine, "Zs", UnicodeCategory.SpaceSeparator };
+                // https://docs.microsoft.com/en-us/dotnet/standard/base-types/character-classes-in-regular-expressions#supported-unicode-general-categories
+
+                yield return new object[] { engine, "L", new[] { UnicodeCategory.UppercaseLetter, UnicodeCategory.LowercaseLetter, UnicodeCategory.TitlecaseLetter, UnicodeCategory.ModifierLetter, UnicodeCategory.OtherLetter } };
+                yield return new object[] { engine, "Lu", new[] { UnicodeCategory.UppercaseLetter } };
+                yield return new object[] { engine, "Ll", new[] { UnicodeCategory.LowercaseLetter } };
+                yield return new object[] { engine, "Lt", new[] { UnicodeCategory.TitlecaseLetter } };
+                yield return new object[] { engine, "Lm", new[] { UnicodeCategory.ModifierLetter } };
+                yield return new object[] { engine, "Lo", new[] { UnicodeCategory.OtherLetter } };
+
+                yield return new object[] { engine, "M", new[] { UnicodeCategory.NonSpacingMark, UnicodeCategory.SpacingCombiningMark, UnicodeCategory.EnclosingMark } };
+                yield return new object[] { engine, "Mn", new[] { UnicodeCategory.NonSpacingMark } };
+                yield return new object[] { engine, "Mc", new[] { UnicodeCategory.SpacingCombiningMark } };
+                yield return new object[] { engine, "Me", new[] { UnicodeCategory.EnclosingMark } };
+
+                yield return new object[] { engine, "N", new[] { UnicodeCategory.DecimalDigitNumber, UnicodeCategory.LetterNumber, UnicodeCategory.OtherNumber } };
+                yield return new object[] { engine, "Nd", new[] { UnicodeCategory.DecimalDigitNumber } };
+                yield return new object[] { engine, "Nl", new[] { UnicodeCategory.LetterNumber } };
+                yield return new object[] { engine, "No", new[] { UnicodeCategory.OtherNumber } };
+
+                yield return new object[] { engine, "P", new[] { UnicodeCategory.ConnectorPunctuation, UnicodeCategory.DashPunctuation, UnicodeCategory.OpenPunctuation, UnicodeCategory.ClosePunctuation, UnicodeCategory.InitialQuotePunctuation, UnicodeCategory.FinalQuotePunctuation, UnicodeCategory.OtherPunctuation } };
+                yield return new object[] { engine, "Pc", new[] { UnicodeCategory.ConnectorPunctuation } };
+                yield return new object[] { engine, "Pd", new[] { UnicodeCategory.DashPunctuation } };
+                yield return new object[] { engine, "Ps", new[] { UnicodeCategory.OpenPunctuation } };
+                yield return new object[] { engine, "Pe", new[] { UnicodeCategory.ClosePunctuation } };
+                yield return new object[] { engine, "Pi", new[] { UnicodeCategory.InitialQuotePunctuation } };
+                yield return new object[] { engine, "Pf", new[] { UnicodeCategory.FinalQuotePunctuation } };
+                yield return new object[] { engine, "Po", new[] { UnicodeCategory.OtherPunctuation } };
+
+                yield return new object[] { engine, "S", new[] { UnicodeCategory.MathSymbol, UnicodeCategory.CurrencySymbol, UnicodeCategory.ModifierSymbol, UnicodeCategory.OtherSymbol } };
+                yield return new object[] { engine, "Sm", new[] { UnicodeCategory.MathSymbol } };
+                yield return new object[] { engine, "Sc", new[] { UnicodeCategory.CurrencySymbol } };
+                yield return new object[] { engine, "Sk", new[] { UnicodeCategory.ModifierSymbol } };
+                yield return new object[] { engine, "So", new[] { UnicodeCategory.OtherSymbol } };
+
+                yield return new object[] { engine, "Z", new[] { UnicodeCategory.SpaceSeparator, UnicodeCategory.LineSeparator, UnicodeCategory.ParagraphSeparator } };
+                yield return new object[] { engine, "Zs", new[] { UnicodeCategory.SpaceSeparator } };
+                yield return new object[] { engine, "Zl", new[] { UnicodeCategory.LineSeparator } };
+                yield return new object[] { engine, "Zp", new[] { UnicodeCategory.ParagraphSeparator } };
+
+                yield return new object[] { engine, "C", new[] { UnicodeCategory.Control, UnicodeCategory.Format, UnicodeCategory.Surrogate, UnicodeCategory.PrivateUse, UnicodeCategory.OtherNotAssigned } };
+                yield return new object[] { engine, "Cc", new[] { UnicodeCategory.Control } };
+                yield return new object[] { engine, "Cf", new[] { UnicodeCategory.Format } };
+                yield return new object[] { engine, "Cs", new[] { UnicodeCategory.Surrogate } };
+                yield return new object[] { engine, "Co", new[] { UnicodeCategory.PrivateUse } };
+                yield return new object[] { engine, "Cn", new[] { UnicodeCategory.OtherNotAssigned } };
             }
         }
 
         [Theory]
         [MemberData(nameof(UnicodeCategoriesInclusionsExpected_MemberData))]
-        public async Task UnicodeCategoriesInclusionsExpected(RegexEngine engine, string generalCategory, UnicodeCategory unicodeCategory)
+        public async Task UnicodeCategoriesInclusionsExpected(RegexEngine engine, string generalCategory, UnicodeCategory[] unicodeCategory) =>
+            await ValidateUnicodeCategoryInclusionExclusion(engine, @$"\p{{{generalCategory}}}", @$"\P{{{generalCategory}}}", unicodeCategory);
+
+        [Theory]
+        [MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))]
+        public async Task LetterOrDigitsInclusionsExpected(RegexEngine engine) =>
+            await ValidateUnicodeCategoryInclusionExclusion(engine, @"[\p{L}\d]", @"[^\p{L}\d]",
+                new[] { UnicodeCategory.UppercaseLetter, UnicodeCategory.LowercaseLetter, UnicodeCategory.TitlecaseLetter, UnicodeCategory.ModifierLetter, UnicodeCategory.OtherLetter, UnicodeCategory.DecimalDigitNumber });
+
+        private async Task ValidateUnicodeCategoryInclusionExclusion(RegexEngine engine, string inclusionPattern, string exclusionPattern, UnicodeCategory[] unicodeCategory)
         {
             Regex r;
 
             char[] allChars = Enumerable.Range(0, char.MaxValue + 1).Select(i => (char)i).ToArray();
-            int expectedInCategory = allChars.Count(c => char.GetUnicodeCategory(c) == unicodeCategory);
+            int expectedInCategory = allChars.Count(c => Array.IndexOf(unicodeCategory, char.GetUnicodeCategory(c)) >= 0);
             int expectedNotInCategory = allChars.Length - expectedInCategory;
 
-            r = await RegexHelpers.GetRegexAsync(engine, @$"\p{{{generalCategory}}}");
+            r = await RegexHelpers.GetRegexAsync(engine, inclusionPattern);
             Assert.Equal(expectedInCategory, r.Matches(string.Concat(allChars)).Count);
 
-            r = await RegexHelpers.GetRegexAsync(engine, (@$"\P{{{generalCategory}}}"));
+            r = await RegexHelpers.GetRegexAsync(engine, exclusionPattern);
             Assert.Equal(expectedNotInCategory, r.Matches(string.Concat(allChars)).Count);
         }