React to changes through ASCIIEncoding and EncoderNLS
authorLevi Broderick <levib@microsoft.com>
Tue, 5 Mar 2019 02:31:41 +0000 (18:31 -0800)
committerLevi Broderick <GrabYourPitchforks@users.noreply.github.com>
Wed, 13 Mar 2019 00:16:30 +0000 (17:16 -0700)
Also create some additional ASCII tests

Commit migrated from https://github.com/dotnet/corefx/commit/d817d68fc6e2fc78624540aa90a060f4d5bf0eee

src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingDecode.cs
src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingDecode.netcoreapp.cs [new file with mode: 0644]
src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingEncode.cs
src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingEncode.netcoreapp.cs [new file with mode: 0644]
src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingGetMaxByteCount.cs
src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingGetMaxCharCount.cs
src/libraries/System.Text.Encoding/tests/Encoder/EncoderConvert2.cs
src/libraries/System.Text.Encoding/tests/NegativeEncodingTests.cs
src/libraries/System.Text.Encoding/tests/System.Text.Encoding.Tests.csproj

index a603c76..7ceb0ea 100644 (file)
@@ -7,7 +7,7 @@ using Xunit;
 
 namespace System.Text.Tests
 {
-    public class ASCIIEncodingDecode
+    public partial class ASCIIEncodingDecode
     {
         public static IEnumerable<object[]> Decode_TestData()
         {
diff --git a/src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingDecode.netcoreapp.cs b/src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingDecode.netcoreapp.cs
new file mode 100644 (file)
index 0000000..2c2983a
--- /dev/null
@@ -0,0 +1,57 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Linq;
+using Xunit;
+
+namespace System.Text.Tests
+{
+    public partial class ASCIIEncodingDecode
+    {
+        [Theory]
+        [InlineData("hello!", 6)]
+        [InlineData("hello\u1234there!", 16)]
+        [InlineData("\ud800\udc00", 10)]
+        public void GetByteCount_WithReplacementFallback(string input, int expectedByteCount)
+        {
+            Encoding encoding = Encoding.GetEncoding("ascii", new EncoderReplacementFallback("abcde"), DecoderFallback.ExceptionFallback);
+            Assert.Equal(expectedByteCount, encoding.GetByteCount(input));
+        }
+
+        [Fact]
+        public void GetByteCount_WithSingleCharNonAsciiReplacementFallback_ValidatesAscii()
+        {
+            // Tests trying to replace one non-ASCII character with another, which should cause
+            // fallback logic to identify the invalid data and abort the operation.
+
+            Encoding encoding = Encoding.GetEncoding("ascii", new EncoderReplacementFallback("\u1234"), DecoderFallback.ExceptionFallback);
+            Assert.Throws<ArgumentException>("chars", () => encoding.GetByteCount("\u0080"));
+        }
+
+        [Theory]
+        [InlineData("hello!", "hello!")]
+        [InlineData("hello\u1234there!", "helloabcdethere!")]
+        [InlineData("\ud800\udc00", "abcdeabcde")]
+        public void GetBytes_WithReplacementFallback(string input, string expectedResult)
+        {
+            Encoding encoding = Encoding.GetEncoding("ascii", new EncoderReplacementFallback("abcde"), DecoderFallback.ExceptionFallback);
+            Assert.Equal(WideToAsciiStr(expectedResult), encoding.GetBytes(input));
+        }
+
+        [Fact]
+        public void GetBytes_WithNonAsciiInput_AndSingleCharNonAsciiReplacementFallback_Throws()
+        {
+            // Tests trying to replace one non-ASCII character with another, which should cause
+            // fallback logic to identify the invalid data and abort the operation.
+
+            Encoding encoding = Encoding.GetEncoding("ascii", new EncoderReplacementFallback("\u1234"), DecoderFallback.ExceptionFallback);
+            Assert.Throws<ArgumentException>("chars", () => encoding.GetBytes("\u0080"));
+        }
+
+        private static byte[] WideToAsciiStr(string input)
+        {
+            return input.Select(ch => (byte)checked((sbyte)ch)).ToArray(); // makes sure each char is 00..7F
+        }
+    }
+}
index d1d7bd6..d62afa7 100644 (file)
@@ -7,7 +7,7 @@ using Xunit;
 
 namespace System.Text.Tests
 {
-    public class ASCIIEncodingEncode
+    public partial class ASCIIEncodingEncode
     {
         public static IEnumerable<object[]> Encode_TestData()
         {
diff --git a/src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingEncode.netcoreapp.cs b/src/libraries/System.Text.Encoding/tests/ASCIIEncoding/ASCIIEncodingEncode.netcoreapp.cs
new file mode 100644 (file)
index 0000000..922d9e0
--- /dev/null
@@ -0,0 +1,91 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Linq;
+using Xunit;
+
+namespace System.Text.Tests
+{
+    public partial class ASCIIEncodingEncode
+    {
+        [Theory]
+        [InlineData("hello!", 6)]
+        [InlineData("hello\u0080there!", 16)]
+        [InlineData("\u00ff\u00ff", 10)]
+        public void GetCharCount_WithReplacementFallback(string input, int expectedCharCount)
+        {
+            Encoding encoding = Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, new DecoderReplacementFallback("abcde"));
+            Assert.Equal(expectedCharCount, encoding.GetCharCount(WideToNarrowStr(input)));
+        }
+
+        [Fact]
+        public void GetCharCount_WithInvalidFallbackBuffer_ValidatesAscii()
+        {
+            // Internal fallback logic should notice that we're about to write out a standalone
+            // surrogate character and should abort the operation.
+
+            Encoding encoding = Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, new StandaloneLowSurrogateDecoderFallback());
+            Assert.Throws<ArgumentException>(() => encoding.GetCharCount(new byte[] { 0x80 }));
+        }
+
+        [Theory]
+        [InlineData("hello!", "hello!")]
+        [InlineData("hello\u0080there!", "helloabcdethere!")]
+        [InlineData("\u00ff\u00ff", "abcdeabcde")]
+        public void GetChars_WithReplacementFallback(string input, string expectedResult)
+        {
+            Encoding encoding = Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, new DecoderReplacementFallback("abcde"));
+            Assert.Equal(expectedResult, encoding.GetChars(WideToNarrowStr(input)));
+        }
+
+        [Fact]
+        public void GetChars_WithNonAsciiInput_AndSingleCharNonAsciiReplacementFallback_Throws()
+        {
+            // Internal fallback logic should notice that we're about to write out a standalone
+            // surrogate character and should abort the operation.
+
+            Encoding encoding = Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, new StandaloneLowSurrogateDecoderFallback());
+            Assert.Throws<ArgumentException>(() => encoding.GetChars(new byte[] { 0x80 }));
+        }
+
+        private static byte[] WideToNarrowStr(string input)
+        {
+            return input.Select(ch => checked((byte)ch)).ToArray(); // makes sure each char is 00..FF
+        }
+
+        private class StandaloneLowSurrogateDecoderFallback : DecoderFallback
+        {
+            public override int MaxCharCount => 1;
+
+            public override DecoderFallbackBuffer CreateFallbackBuffer()
+            {
+                return new InnerFallbackBuffer();
+            }
+
+            private class InnerFallbackBuffer : DecoderFallbackBuffer
+            {
+                private int _remaining;
+
+                public override int Remaining => _remaining;
+
+                public override bool Fallback(byte[] bytesUnknown, int index)
+                {
+                    _remaining = 1;
+                    return true;
+                }
+
+                public override char GetNextChar()
+                {
+                    // Return a standalone low surrogate
+                    return (--_remaining >= 0) ? '\udc00' : default;
+                }
+
+                public override bool MovePrevious()
+                {
+                    throw new NotImplementedException();
+                }
+            }
+        }
+    }
+}
index 1d29ec8..06c48da 100644 (file)
@@ -15,6 +15,48 @@ namespace System.Text.Tests
         public void GetMaxByteCount(int charCount)
         {
             Assert.Equal(charCount + 1, new ASCIIEncoding().GetMaxByteCount(charCount));
+
+            // Now test the input for an Encoding which has a zero or negative-length EncoderFallback.MaxCharCount.
+
+            Assert.Equal(charCount + 1, Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, DecoderFallback.ExceptionFallback).GetMaxByteCount(charCount));
+            Assert.Equal(charCount + 1, Encoding.GetEncoding("ascii", new CustomLengthEncoderFallback(-5), DecoderFallback.ExceptionFallback).GetMaxByteCount(charCount));
+        }
+
+        [Theory]
+        [InlineData(0, 5)]
+        [InlineData(10, 55)]
+        [InlineData(10_000, 50_005)]
+        public void GetMaxByteCount_WithLongEncoderFallback(int charCount, int expectedMaxByteCount)
+        {
+            Encoding asciiEncoding = Encoding.GetEncoding("ascii", new EncoderReplacementFallback("abcde"), DecoderFallback.ExceptionFallback);
+            Assert.Equal(expectedMaxByteCount, asciiEncoding.GetMaxByteCount(charCount));
+        }
+
+        [Theory]
+        [InlineData(-1)]
+        [InlineData(int.MaxValue)]
+        public void GetMaxByteCount_WithDefaultEncoder_InvalidArg(int charCount)
+        {
+            Assert.Throws<ArgumentOutOfRangeException>("charCount", () => Encoding.ASCII.GetMaxByteCount(charCount));
+        }
+
+        [Fact]
+        public void GetMaxByteCount_Overflow_WithLongEncoderFallbackMaxCharCount()
+        {
+            Encoding asciiEncoding = Encoding.GetEncoding("ascii", new CustomLengthEncoderFallback(1_000_000), DecoderFallback.ExceptionFallback);
+            Assert.Throws<ArgumentOutOfRangeException>("charCount", () => asciiEncoding.GetMaxByteCount(5_000_000));
+        }
+
+        private class CustomLengthEncoderFallback : EncoderFallback
+        {
+            public CustomLengthEncoderFallback(int maxCharCount) { MaxCharCount = maxCharCount; }
+
+            public override int MaxCharCount { get; }
+
+            public override EncoderFallbackBuffer CreateFallbackBuffer()
+            {
+                throw new NotImplementedException();
+            }
         }
     }
 }
index 11c7641..18b4784 100644 (file)
@@ -15,6 +15,46 @@ namespace System.Text.Tests
         public void GetMaxCharCount(int byteCount)
         {
             Assert.Equal(byteCount, new ASCIIEncoding().GetMaxCharCount(byteCount));
+
+            // Now test the input for an Encoding which has a zero or negative-length DecoderFallback.MaxCharCount.
+
+            Assert.Equal(byteCount, Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, DecoderFallback.ExceptionFallback).GetMaxCharCount(byteCount));
+            Assert.Equal(byteCount, Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, new CustomLengthDecoderFallback(-5)).GetMaxCharCount(byteCount));
+        }
+
+        [Theory]
+        [InlineData(0, 0)]
+        [InlineData(10, 50)]
+        [InlineData(10_000, 50_000)]
+        public void GetMaxCharCount_WithLongDecoderFallback(int byteCount, int expectedMaxCharCount)
+        {
+            Encoding asciiEncoding = Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, new DecoderReplacementFallback("abcde"));
+            Assert.Equal(expectedMaxCharCount, asciiEncoding.GetMaxCharCount(byteCount));
+        }
+
+        [Fact]
+        public void GetMaxCharCount_WithDefaultDecoder_InvalidArg()
+        {
+            Assert.Throws<ArgumentOutOfRangeException>("byteCount", () => Encoding.ASCII.GetMaxCharCount(-1));
+        }
+
+        [Fact]
+        public void GetMaxCharCount_Overflow_WithLongDecoderFallbackMaxCharCount()
+        {
+            Encoding asciiEncoding = Encoding.GetEncoding("ascii", EncoderFallback.ExceptionFallback, new CustomLengthDecoderFallback(1_000_000));
+            Assert.Throws<ArgumentOutOfRangeException>("byteCount", () => asciiEncoding.GetMaxCharCount(5_000_000));
+        }
+
+        private class CustomLengthDecoderFallback : DecoderFallback
+        {
+            public CustomLengthDecoderFallback(int maxCharCount) { MaxCharCount = maxCharCount; }
+
+            public override int MaxCharCount { get; }
+
+            public override DecoderFallbackBuffer CreateFallbackBuffer()
+            {
+                throw new NotImplementedException();
+            }
         }
     }
 }
index e6e0a81..95f1813 100644 (file)
@@ -210,27 +210,28 @@ namespace System.Text.Tests
         
         // Call Convert to convert partial of a Unicode character array with UTF8 encoder
         [Fact]
+        [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "coreclr #23020 is not fixed in netfx.")]
         public void EncoderUTF8ConvertUnicodeCharArrayPartial()
         {
             char[] chars = "\uD83D\uDE01Test".ToCharArray();
             byte[] bytes = new byte[chars.Length * 2];
             Encoder encoder = Encoding.UTF8.GetEncoder();
 
-            VerificationHelper(encoder, chars, 0, 1, bytes, 0, 0, true, 1, 0, expectedCompleted: true);
-            VerificationHelper(encoder, chars, 0, 1, bytes, 0, 3, false, 1, 3, expectedCompleted: true);
-            VerificationHelper(encoder, chars, 0, 2, bytes, 0, 7, false, 2, 7, expectedCompleted: true);
+            VerificationHelper(encoder, chars, 0, 1, bytes, 0, 3, true, 1, 3, expectedCompleted: true);
+            VerificationHelper(encoder, chars, 0, 2, bytes, 0, 7, false, 2, 4, expectedCompleted: true);
             VerificationHelper(encoder, chars, 0, 4, bytes, 0, 6, false, 4, 6, expectedCompleted: true);
             VerificationHelper(encoder, chars, 0, 4, bytes, 0, 6, true, 4, 6, expectedCompleted: true);
             VerificationHelper(encoder, chars, 2, 2, bytes, 0, 2, true, 2, 2, expectedCompleted: true);
             VerificationHelper(encoder, chars, 1, 3, bytes, 1, 3, false, 1, 3, expectedCompleted: false);
             VerificationHelper(encoder, chars, 1, 3, bytes, 1, 5, true, 3, 5, expectedCompleted: true);
 
-            VerificationHelper(encoder, chars, 0, 1, bytes, 0, bytes.Length, false, 1, 0, expectedCompleted: true);
+            VerificationHelper(encoder, chars, 0, 1, bytes, 0, bytes.Length, false, 1, 0, expectedCompleted: false);
             VerificationHelper(encoder, chars, 1, 1, bytes, 0, bytes.Length, true, 1, 4, expectedCompleted: true);
         }
  
         // Call Convert to convert partial of a ASCII+Unicode character array with ASCII encoder
         [Fact]
+        [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "coreclr #23020 is not fixed in netfx.")]
         public void EncoderASCIIConvertMixedASCIIUnicodeCharArrayPartial()
         {
             char[] chars = "T\uD83D\uDE01est".ToCharArray();
@@ -238,8 +239,9 @@ namespace System.Text.Tests
             Encoder encoder = Encoding.ASCII.GetEncoder();
 
             VerificationHelper(encoder, chars, 0, 1, bytes, 0, 1, true, 1, 1, expectedCompleted: true);
-            VerificationHelper(encoder, chars, 0, 2, bytes, 0, 1, false, 1, 1, expectedCompleted: false);
-            VerificationHelper(encoder, chars, 1, 2, bytes, 0, 2, false, 2, 2, expectedCompleted: true);
+            VerificationHelper(encoder, chars, 0, 4, bytes, 0, 1, false, 3, 1, expectedCompleted: false);
+            VerificationHelper(encoder, chars, 3, 1, bytes, 0, 2, false, 0, 2, expectedCompleted: false);
+            VerificationHelper(encoder, chars, 3, 1, bytes, 0, 2, false, 1, 1, expectedCompleted: true);
             VerificationHelper(encoder, chars, 0, 5, bytes, 0, 5, false, 5, 5, expectedCompleted: true);
             VerificationHelper(encoder, chars, 0, 4, bytes, 0, 4, true, 4, 4, expectedCompleted: true);
             VerificationHelper(encoder, chars, 2, 2, bytes, 0, 2, true, 2, 2, expectedCompleted: true);
@@ -252,6 +254,7 @@ namespace System.Text.Tests
         
         // Call Convert to convert partial of a ASCII+Unicode character array with UTF8 encoder
         [Fact]
+        [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "coreclr #23020 is not fixed in netfx.")]
         public void EncoderUTF8ConvertMixedASCIIUnicodeCharArrayPartial()
         {
             char[] chars = "T\uD83D\uDE01est".ToCharArray();
@@ -259,15 +262,17 @@ namespace System.Text.Tests
             Encoder encoder = Encoding.UTF8.GetEncoder();
 
             VerificationHelper(encoder, chars, 0, 1, bytes, 0, 1, true, 1, 1, expectedCompleted: true);
-            VerificationHelper(encoder, chars, 0, 2, bytes, 0, 1, false, 2, 1, expectedCompleted: true);
-            VerificationHelper(encoder, chars, 1, 2, bytes, 0, 7, false, 2, 7, expectedCompleted: true);
+            VerificationHelper(encoder, chars, 0, 2, bytes, 0, 1, false, 2, 1, expectedCompleted: false);
+            VerificationHelper(encoder, chars, 2, 1, bytes, 0, 5, false, 1, 4, expectedCompleted: true);
+            VerificationHelper(encoder, chars, 1, 2, bytes, 0, 7, false, 2, 4, expectedCompleted: true);
             VerificationHelper(encoder, chars, 0, 5, bytes, 0, 7, false, 5, 7, expectedCompleted: true);
             VerificationHelper(encoder, chars, 0, 4, bytes, 0, 6, true, 4, 6, expectedCompleted: true);
             VerificationHelper(encoder, chars, 2, 2, bytes, 0, 3, true, 1, 3, expectedCompleted: false);
             VerificationHelper(encoder, chars, 1, 3, bytes, 1, 5, false, 3, 5, expectedCompleted: true);
             VerificationHelper(encoder, chars, 1, 3, bytes, 1, 5, true, 3, 5, expectedCompleted: true);
 
-            VerificationHelper(encoder, chars, 0, 2, bytes, 0, bytes.Length, false, 2, 1, expectedCompleted: true);
+            VerificationHelper(encoder, chars, 0, 2, bytes, 0, bytes.Length, false, 2, 1, expectedCompleted: false);
+            VerificationHelper(encoder, chars, 2, 2, bytes, 0, bytes.Length, false, 2, 5, expectedCompleted: true);
             VerificationHelper(encoder, chars, 1, 1, bytes, 0, bytes.Length, true, 1, 3, expectedCompleted: true);
         }
 
@@ -279,7 +284,7 @@ namespace System.Text.Tests
             int bytesUsed;
             bool completed;
 
-            encoder.Convert(chars, charIndex, charCount, bytes, byteIndex, byteCount, false, out charsUsed, out bytesUsed, out completed);
+            encoder.Convert(chars, charIndex, charCount, bytes, byteIndex, byteCount, flush, out charsUsed, out bytesUsed, out completed);
             Assert.Equal(expectedCharsUsed, charsUsed);
             Assert.Equal(expectedBytesUsed, bytesUsed);
             Assert.Equal(expectedCompleted, completed);
index 9585f66..2718f6f 100644 (file)
@@ -16,13 +16,18 @@ namespace System.Text.Tests
             yield return new object[] { new UnicodeEncoding(true, false) };
             yield return new object[] { new UnicodeEncoding(true, true) };
             yield return new object[] { new UnicodeEncoding(true, true) };
+            yield return new object[] { Encoding.BigEndianUnicode };
+            yield return new object[] { Encoding.Unicode };
             yield return new object[] { new UTF7Encoding(true) };
             yield return new object[] { new UTF7Encoding(false) };
+            yield return new object[] { Encoding.UTF7 };
             yield return new object[] { new UTF8Encoding(true, true) };
             yield return new object[] { new UTF8Encoding(false, true) };
             yield return new object[] { new UTF8Encoding(true, false) };
             yield return new object[] { new UTF8Encoding(false, false) };
+            yield return new object[] { Encoding.UTF8 };
             yield return new object[] { new ASCIIEncoding() };
+            yield return new object[] { Encoding.ASCII };
             yield return new object[] { new UTF32Encoding(true, true, true) };
             yield return new object[] { new UTF32Encoding(true, true, false) };
             yield return new object[] { new UTF32Encoding(true, false, false) };
@@ -31,6 +36,7 @@ namespace System.Text.Tests
             yield return new object[] { new UTF32Encoding(false, true, false) };
             yield return new object[] { new UTF32Encoding(false, false, false) };
             yield return new object[] { new UTF32Encoding(false, false, true) };
+            yield return new object[] { Encoding.UTF32 };
             yield return new object[] { Encoding.GetEncoding("latin1") };
         }
 
index b57fdc7..fc534d1 100644 (file)
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <ProjectGuid>{3BB28F2F-51DF-49A3-A0BF-E1C5C0D7E3E0}</ProjectGuid>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
@@ -8,6 +8,8 @@
   <ItemGroup>
     <Compile Include="ASCIIEncoding\ASCIIEncodingEncode.cs" />
     <Compile Include="ASCIIEncoding\ASCIIEncodingDecode.cs" />
+    <Compile Include="ASCIIEncoding\ASCIIEncodingEncode.netcoreapp.cs" Condition="'$(TargetGroup)'=='netcoreapp'" />
+    <Compile Include="ASCIIEncoding\ASCIIEncodingDecode.netcoreapp.cs" Condition="'$(TargetGroup)'=='netcoreapp'" />
     <Compile Include="ASCIIEncoding\ASCIIEncodingGetDecoder.cs" />
     <Compile Include="ASCIIEncoding\ASCIIEncodingGetEncoder.cs" />
     <Compile Include="ASCIIEncoding\ASCIIEncodingGetMaxByteCount.cs" />