Handle non-ASCII strings in GetNonRandomizedHashCodeOrdinalIgnoreCase (#44688)
authorEgor Bogatov <egorbo@gmail.com>
Sat, 21 Nov 2020 23:01:32 +0000 (02:01 +0300)
committerGitHub <noreply@github.com>
Sat, 21 Nov 2020 23:01:32 +0000 (15:01 -0800)
* Fix GetNonRandomizedHashCodeOrdinalIgnoreCase

* Add a test

Co-authored-by: Levi Broderick <levib@microsoft.com>
src/libraries/System.Collections/tests/Generic/Dictionary/Dictionary.Tests.cs
src/libraries/System.Collections/tests/Generic/Dictionary/HashCollisionScenarios/OutOfBoundsRegression.cs
src/libraries/System.Private.CoreLib/src/System/Collections/Generic/RandomizedStringEqualityComparer.cs
src/libraries/System.Private.CoreLib/src/System/String.Comparison.cs

index ee9957b..653f6c7 100644 (file)
@@ -337,6 +337,22 @@ namespace System.Collections.Tests
             AssertExtensions.Throws<ArgumentException>(null, () => new Dictionary<string, int>(source, StringComparer.OrdinalIgnoreCase));
         }
 
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotInvariantGlobalization))]
+        // https://github.com/dotnet/runtime/issues/44681
+        public void DictionaryOrdinalIgnoreCaseCyrillicKeys()
+        {
+            const string Lower = "абвгдеёжзийклмнопрстуфхцчшщьыъэюя";
+            const string Higher = "АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЬЫЪЭЮЯ";
+
+            var dictionary = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+
+            for (int i = 0; i < Lower.Length; i++)
+            {
+                dictionary[Lower[i].ToString()] = i;
+                Assert.Equal(i, dictionary[Higher[i].ToString()]);
+            }
+        }
+
         public static IEnumerable<object[]> CopyConstructorStringComparerData
         {
             get
index 934e57a..eb7a7bf 100644 (file)
@@ -54,56 +54,56 @@ namespace System.Collections.Tests
 
             RunDictionaryTest(
                 equalityComparer: null,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalComparerType);\r
 
             // EqualityComparer<string>.Default comparer
 
             RunDictionaryTest(
                 equalityComparer: EqualityComparer<string>.Default,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalComparerType);\r
 
             // Ordinal comparer
 
             RunDictionaryTest(
                 equalityComparer: StringComparer.Ordinal,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: StringComparer.Ordinal.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: StringComparer.Ordinal,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalComparerType);\r
 
             // OrdinalIgnoreCase comparer
 
             RunDictionaryTest(
                 equalityComparer: StringComparer.OrdinalIgnoreCase,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalIgnoreCaseComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: StringComparer.OrdinalIgnoreCase.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalIgnoreCaseComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalIgnoreCaseComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: StringComparer.OrdinalIgnoreCase,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalIgnoreCaseComparerType);\r
 
             // linguistic comparer (not optimized)
 
             RunDictionaryTest(
                 equalityComparer: StringComparer.InvariantCulture,
-                expectedInternalComparerBeforeCollisionThreshold: StringComparer.InvariantCulture.GetType(),
-                expectedPublicComparerBeforeCollisionThreshold: StringComparer.InvariantCulture.GetType(),
-                expectedComparerAfterCollisionThreshold: StringComparer.InvariantCulture.GetType());
+                expectedInternalComparerTypeBeforeCollisionThreshold: StringComparer.InvariantCulture.GetType(),\r
+                expectedPublicComparerBeforeCollisionThreshold: StringComparer.InvariantCulture,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: StringComparer.InvariantCulture.GetType());\r
 
             static void RunDictionaryTest(
                 IEqualityComparer<string> equalityComparer,
-                Type expectedInternalComparerBeforeCollisionThreshold,
-                Type expectedPublicComparerBeforeCollisionThreshold,
-                Type expectedComparerAfterCollisionThreshold)
+                Type expectedInternalComparerTypeBeforeCollisionThreshold,\r
+                IEqualityComparer<string> expectedPublicComparerBeforeCollisionThreshold,\r
+                Type expectedInternalComparerTypeAfterCollisionThreshold)\r
             {
                 RunCollectionTestCommon(
                     () => new Dictionary<string, object>(equalityComparer),
                     (dictionary, key) => dictionary.Add(key, null),
                     (dictionary, key) => dictionary.ContainsKey(key),
                     dictionary => dictionary.Comparer,
-                    expectedInternalComparerBeforeCollisionThreshold,
+                    expectedInternalComparerTypeBeforeCollisionThreshold,\r
                     expectedPublicComparerBeforeCollisionThreshold,
-                    expectedComparerAfterCollisionThreshold);
+                    expectedInternalComparerTypeAfterCollisionThreshold);\r
             }
         }
 
@@ -119,56 +119,56 @@ namespace System.Collections.Tests
 
             RunHashSetTest(
                 equalityComparer: null,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalComparerType);\r
 
             // EqualityComparer<string>.Default comparer
 
             RunHashSetTest(
                 equalityComparer: EqualityComparer<string>.Default,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: EqualityComparer<string>.Default,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalComparerType);\r
 
             // Ordinal comparer
 
             RunHashSetTest(
                 equalityComparer: StringComparer.Ordinal,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: StringComparer.Ordinal.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: StringComparer.Ordinal,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalComparerType);\r
 
             // OrdinalIgnoreCase comparer
 
             RunHashSetTest(
                 equalityComparer: StringComparer.OrdinalIgnoreCase,
-                expectedInternalComparerBeforeCollisionThreshold: nonRandomizedOrdinalIgnoreCaseComparerType,
-                expectedPublicComparerBeforeCollisionThreshold: StringComparer.OrdinalIgnoreCase.GetType(),
-                expectedComparerAfterCollisionThreshold: randomizedOrdinalIgnoreCaseComparerType);
+                expectedInternalComparerTypeBeforeCollisionThreshold: nonRandomizedOrdinalIgnoreCaseComparerType,\r
+                expectedPublicComparerBeforeCollisionThreshold: StringComparer.OrdinalIgnoreCase,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: randomizedOrdinalIgnoreCaseComparerType);\r
 
             // linguistic comparer (not optimized)
 
             RunHashSetTest(
                 equalityComparer: StringComparer.InvariantCulture,
-                expectedInternalComparerBeforeCollisionThreshold: StringComparer.InvariantCulture.GetType(),
-                expectedPublicComparerBeforeCollisionThreshold: StringComparer.InvariantCulture.GetType(),
-                expectedComparerAfterCollisionThreshold: StringComparer.InvariantCulture.GetType());
+                expectedInternalComparerTypeBeforeCollisionThreshold: StringComparer.InvariantCulture.GetType(),\r
+                expectedPublicComparerBeforeCollisionThreshold: StringComparer.InvariantCulture,\r
+                expectedInternalComparerTypeAfterCollisionThreshold: StringComparer.InvariantCulture.GetType());\r
 
             static void RunHashSetTest(
                 IEqualityComparer<string> equalityComparer,
-                Type expectedInternalComparerBeforeCollisionThreshold,
-                Type expectedPublicComparerBeforeCollisionThreshold,
-                Type expectedComparerAfterCollisionThreshold)
+                Type expectedInternalComparerTypeBeforeCollisionThreshold,\r
+                IEqualityComparer<string> expectedPublicComparerBeforeCollisionThreshold,\r
+                Type expectedInternalComparerTypeAfterCollisionThreshold)\r
             {
                 RunCollectionTestCommon(
                     () => new HashSet<string>(equalityComparer),
                     (set, key) => Assert.True(set.Add(key)),
                     (set, key) => set.Contains(key),
                     set => set.Comparer,
-                    expectedInternalComparerBeforeCollisionThreshold,
+                    expectedInternalComparerTypeBeforeCollisionThreshold,\r
                     expectedPublicComparerBeforeCollisionThreshold,
-                    expectedComparerAfterCollisionThreshold);
+                    expectedInternalComparerTypeAfterCollisionThreshold);\r
             }
         }
 
@@ -177,24 +177,18 @@ namespace System.Collections.Tests
             Action<TCollection, string> addKeyCallback,
             Func<TCollection, string, bool> containsKeyCallback,
             Func<TCollection, IEqualityComparer<string>> getComparerCallback,
-            Type expectedInternalComparerBeforeCollisionThreshold,
-            Type expectedPublicComparerBeforeCollisionThreshold,
-            Type expectedComparerAfterCollisionThreshold)
+            Type expectedInternalComparerTypeBeforeCollisionThreshold,\r
+            IEqualityComparer<string> expectedPublicComparerBeforeCollisionThreshold,\r
+            Type expectedInternalComparerTypeAfterCollisionThreshold)\r
         {
             TCollection collection = collectionFactory();
             List<string> allKeys = new List<string>();
 
-            const int StartOfRange = 0xE020; // use the Unicode Private Use range to avoid accidentally creating strings that really do compare as equal OrdinalIgnoreCase
-            const int Stride = 0x40; // to ensure we don't accidentally reset the 0x20 bit of the seed, which is used to negate OrdinalIgnoreCase effects
-
             // First, go right up to the collision threshold, but don't exceed it.
 
             for (int i = 0; i < 100; i++)
             {
-                string newKey = GenerateCollidingString(i * Stride + StartOfRange);
-                Assert.Equal(0, _lazyGetNonRandomizedHashCodeDel.Value(newKey)); // ensure has a zero hash code Ordinal
-                Assert.Equal(0x24716ca0, _lazyGetNonRandomizedOrdinalIgnoreCaseHashCodeDel.Value(newKey)); // ensure has a zero hash code OrdinalIgnoreCase
-
+                string newKey = _collidingStrings[i];\r
                 addKeyCallback(collection, newKey);
                 allKeys.Add(newKey);
             }
@@ -202,15 +196,18 @@ namespace System.Collections.Tests
             FieldInfo internalComparerField = collection.GetType().GetField("_comparer", BindingFlags.NonPublic | BindingFlags.Instance);
             Assert.NotNull(internalComparerField);
 
-            Assert.Equal(expectedInternalComparerBeforeCollisionThreshold, internalComparerField.GetValue(collection)?.GetType());
-            Assert.Equal(expectedPublicComparerBeforeCollisionThreshold, getComparerCallback(collection).GetType());
+            IEqualityComparer<string> actualInternalComparerBeforeCollisionThreshold = (IEqualityComparer<string>)internalComparerField.GetValue(collection);\r
+            ValidateBehaviorOfInternalComparerVsPublicComparer(actualInternalComparerBeforeCollisionThreshold, expectedPublicComparerBeforeCollisionThreshold);\r
+\r
+            Assert.Equal(expectedInternalComparerTypeBeforeCollisionThreshold, actualInternalComparerBeforeCollisionThreshold?.GetType());\r
+            Assert.Equal(expectedPublicComparerBeforeCollisionThreshold, getComparerCallback(collection));\r
 
             // Now exceed the collision threshold, which should rebucket entries.
             // Continue adding a few more entries to ensure we didn't corrupt internal state.
 
             for (int i = 100; i < 110; i++)
             {
-                string newKey = GenerateCollidingString(i * Stride + StartOfRange);
+                string newKey = _collidingStrings[i];\r
                 Assert.Equal(0, _lazyGetNonRandomizedHashCodeDel.Value(newKey)); // ensure has a zero hash code Ordinal
                 Assert.Equal(0x24716ca0, _lazyGetNonRandomizedOrdinalIgnoreCaseHashCodeDel.Value(newKey)); // ensure has a zero hash code OrdinalIgnoreCase
 
@@ -218,8 +215,11 @@ namespace System.Collections.Tests
                 allKeys.Add(newKey);
             }
 
-            Assert.Equal(expectedComparerAfterCollisionThreshold, internalComparerField.GetValue(collection)?.GetType());
-            Assert.Equal(expectedPublicComparerBeforeCollisionThreshold, getComparerCallback(collection).GetType()); // shouldn't change this return value after collision threshold met
+            IEqualityComparer<string> actualInternalComparerAfterCollisionThreshold = (IEqualityComparer<string>)internalComparerField.GetValue(collection);\r
+            ValidateBehaviorOfInternalComparerVsPublicComparer(actualInternalComparerAfterCollisionThreshold, expectedPublicComparerBeforeCollisionThreshold);\r
+\r
+            Assert.Equal(expectedInternalComparerTypeAfterCollisionThreshold, actualInternalComparerAfterCollisionThreshold?.GetType());\r
+            Assert.Equal(expectedPublicComparerBeforeCollisionThreshold, getComparerCallback(collection)); // shouldn't change this return value after collision threshold met\r
 
             // And validate that all strings are present in the dictionary.
 
@@ -235,7 +235,7 @@ namespace System.Collections.Tests
             ((ISerializable)collection).GetObjectData(si, new StreamingContext());
 
             object serializedComparer = si.GetValue("Comparer", typeof(IEqualityComparer<string>));
-            Assert.Equal(expectedPublicComparerBeforeCollisionThreshold, serializedComparer.GetType());
+            Assert.Equal(expectedPublicComparerBeforeCollisionThreshold, serializedComparer);\r
         }
 
         private static Lazy<Func<string, int>> _lazyGetNonRandomizedHashCodeDel = new Lazy<Func<string, int>>(
@@ -244,27 +244,63 @@ namespace System.Collections.Tests
         private static Lazy<Func<string, int>> _lazyGetNonRandomizedOrdinalIgnoreCaseHashCodeDel = new Lazy<Func<string, int>>(
             () => GetStringHashCodeOpenDelegate("GetNonRandomizedHashCodeOrdinalIgnoreCase"));
 
-        // Generates a string with a well-known non-randomized hash code:
-        // - string.GetNonRandomizedHashCode returns 0.
-        // - string.GetNonRandomizedHashCodeOrdinalIgnoreCase returns 0x24716ca0.
-        // Provide a different seed to produce a different string.
-        private static string GenerateCollidingString(int seed)
+        // n.b., must be initialized *after* delegate fields above\r
+        private static readonly List<string> _collidingStrings = GenerateCollidingStrings(110);\r
+\r
+        private static List<string> GenerateCollidingStrings(int count)\r
         {
-            return string.Create(8, seed, (span, seed) =>
+            const int StartOfRange = 0xE020; // use the Unicode Private Use range to avoid accidentally creating strings that really do compare as equal OrdinalIgnoreCase\r
+            const int Stride = 0x40; // to ensure we don't accidentally reset the 0x20 bit of the seed, which is used to negate OrdinalIgnoreCase effects\r
+\r
+            int currentSeed = StartOfRange;\r
+\r
+            List<string> collidingStrings = new List<string>(count);\r
+            while (collidingStrings.Count < count)\r
             {
-                Span<byte> asBytes = MemoryMarshal.AsBytes(span);
-
-                uint hash1 = (5381 << 16) + 5381;
-                uint hash2 = BitOperations.RotateLeft(hash1, 5) + hash1;
-
-                MemoryMarshal.Write(asBytes, ref seed);
-                MemoryMarshal.Write(asBytes.Slice(4), ref hash2); // set hash2 := 0 (for Ordinal)
-
-                hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1) ^ (uint)seed;
-                hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1);
-
-                MemoryMarshal.Write(asBytes.Slice(8), ref hash1); // set hash1 := 0 (for Ordinal)
-            });
+                if (currentSeed > ushort.MaxValue)\r
+                {\r
+                    throw new Exception($"Couldn't create enough colliding strings? Created {collidingStrings.Count}, needed {count}.");\r
+                }\r
+
+                string candidate = GenerateCollidingStringCandidate(currentSeed);\r
+
+                int ordinalHashCode = _lazyGetNonRandomizedHashCodeDel.Value(candidate);\r
+                Assert.Equal(0, ordinalHashCode); // ensure has a zero hash code Ordinal\r
+
+                int ordinalIgnoreCaseHashCode = _lazyGetNonRandomizedOrdinalIgnoreCaseHashCodeDel.Value(candidate);\r
+                if (ordinalIgnoreCaseHashCode == 0x24716ca0) // ensure has a zero hash code OrdinalIgnoreCase (might not have one)\r
+                {\r
+                    collidingStrings.Add(candidate); // success!\r
+                }\r
+
+                currentSeed += Stride;\r
+            }\r
+\r
+            return collidingStrings;\r
+\r
+            // Generates a possible string with a well-known non-randomized hash code:\r
+            // - string.GetNonRandomizedHashCode returns 0.\r
+            // - string.GetNonRandomizedHashCodeOrdinalIgnoreCase returns 0x24716ca0.\r
+            // Provide a different seed to produce a different string.\r
+            // Caller must check OrdinalIgnoreCase hash code to ensure correctness.\r
+            static string GenerateCollidingStringCandidate(int seed)\r
+            {\r
+                return string.Create(8, seed, (span, seed) =>\r
+                {\r
+                    Span<byte> asBytes = MemoryMarshal.AsBytes(span);\r
+\r
+                    uint hash1 = (5381 << 16) + 5381;\r
+                    uint hash2 = BitOperations.RotateLeft(hash1, 5) + hash1;\r
+\r
+                    MemoryMarshal.Write(asBytes, ref seed);\r
+                    MemoryMarshal.Write(asBytes.Slice(4), ref hash2); // set hash2 := 0 (for Ordinal)\r
+\r
+                    hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1) ^ (uint)seed;\r
+                    hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1);\r
+\r
+                    MemoryMarshal.Write(asBytes.Slice(8), ref hash1); // set hash1 := 0 (for Ordinal)\r
+                });\r
+            }\r
         }
 
         private static Func<string, int> GetStringHashCodeOpenDelegate(string methodName)
@@ -274,5 +310,44 @@ namespace System.Collections.Tests
 
             return method.CreateDelegate<Func<string, int>>(target: null); // create open delegate unbound to 'this'
         }
+\r
+        private static void ValidateBehaviorOfInternalComparerVsPublicComparer(IEqualityComparer<string> internalComparer, IEqualityComparer<string> publicComparer)\r
+        {\r
+            // This helper ensures that when we substitute one of our internal comparers\r
+            // in place of the expected public comparer, the internal comparer's Equals\r
+            // and GetHashCode behavior are consistent with the public comparer's.\r
+\r
+            if (internalComparer is null)\r
+            {\r
+                internalComparer = EqualityComparer<string>.Default;\r
+            }\r
+            if (publicComparer is null)\r
+            {\r
+                publicComparer = EqualityComparer<string>.Default;\r
+            }\r
+\r
+            foreach (var pair in new[] {\r
+                ("Hello", "Hello"), // exactly equal\r
+                ("Hello", "Goodbye"), // not equal at all\r
+                ("Hello", "hello"), // case-insensitive equal\r
+                ("Hello", "He\u200dllo"), // equal under linguistic comparer\r
+                ("Hello", "HE\u200dLLO"), // equal under case-insensitive linguistic comparer\r
+                ("абвгдеёжзийклмнопрстуфхцчшщьыъэюя", "АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЬЫЪЭЮЯ"), // Cyrillic, case-insensitive equal\r
+            })\r
+            {\r
+                bool arePairElementsExpectedEqual = publicComparer.Equals(pair.Item1, pair.Item2);\r
+                Assert.Equal(arePairElementsExpectedEqual, internalComparer.Equals(pair.Item1, pair.Item2));\r
+\r
+                bool areInternalHashCodesEqual = internalComparer.GetHashCode(pair.Item1) == internalComparer.GetHashCode(pair.Item2);\r
+                if (arePairElementsExpectedEqual)\r
+                {\r
+                    Assert.True(areInternalHashCodesEqual);\r
+                }\r
+                else if (!areInternalHashCodesEqual)\r
+                {\r
+                    Assert.False(arePairElementsExpectedEqual);\r
+                }\r
+            }\r
+        }\r
     }
 }
index 168959d..30db604 100644 (file)
@@ -87,31 +87,6 @@ namespace System.Collections.Generic
                     return 0;
                 }
 
-                // The Ordinal version of Marvin32 operates over bytes, so convert
-                // char count -> byte count. Guaranteed not to integer overflow.
-                return Marvin.ComputeHash32(
-                    ref Unsafe.As<char, byte>(ref obj.GetRawStringData()),
-                    (uint)obj.Length * sizeof(char),
-                    _seed.p0, _seed.p1);
-            }
-        }
-
-        private sealed class RandomizedOrdinalIgnoreCaseComparer : RandomizedStringEqualityComparer
-        {
-            internal RandomizedOrdinalIgnoreCaseComparer(IEqualityComparer<string?> underlyingComparer)
-                : base(underlyingComparer)
-            {
-            }
-
-            public override bool Equals(string? x, string? y) => string.EqualsOrdinalIgnoreCase(x, y);
-
-            public override int GetHashCode(string? obj)
-            {
-                if (obj is null)
-                {
-                    return 0;
-                }
-
                 // The OrdinalIgnoreCase version of Marvin32 operates over chars,
                 // so pass in the char count directly.
                 return Marvin.ComputeHash32OrdinalIgnoreCase(
index 44deea8..bab6ff4 100644 (file)
@@ -1,11 +1,13 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Buffers;
 using System.Diagnostics;
 using System.Globalization;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Text.Unicode;
 
 using Internal.Runtime.CompilerServices;
 
@@ -834,42 +836,96 @@ namespace System
             }
         }
 
-        // Use this if and only if 'Denial of Service' attacks are not a concern (i.e. never used for free-form user input),
-        // or are otherwise mitigated
         internal unsafe int GetNonRandomizedHashCodeOrdinalIgnoreCase()
         {
+            uint hash1 = (5381 << 16) + 5381;
+            uint hash2 = hash1;
+
             fixed (char* src = &_firstChar)
             {
                 Debug.Assert(src[this.Length] == '\0', "src[this.Length] == '\\0'");
-                Debug.Assert(((int)src) % 4 == 0, "Managed string should start at 4 bytes boundary");
+                Debug.Assert(((int) src) % 4 == 0, "Managed string should start at 4 bytes boundary");
 
-                uint hash1 = (5381 << 16) + 5381;
-                uint hash2 = hash1;
-
-                uint* ptr = (uint*)src;
+                uint* ptr = (uint*) src;
                 int length = this.Length;
 
                 // We "normalize to lowercase" every char by ORing with 0x0020. This casts
                 // a very wide net because it will change, e.g., '^' to '~'. But that should
                 // be ok because we expect this to be very rare in practice.
-
                 const uint NormalizeToLowercase = 0x0020_0020u; // valid both for big-endian and for little-endian
 
                 while (length > 2)
                 {
+                    uint p0 = ptr[0];
+                    uint p1 = ptr[1];
+                    if (!Utf16Utility.AllCharsInUInt32AreAscii(p0 | p1))
+                    {
+                        goto NotAscii;
+                    }
+
                     length -= 4;
                     // Where length is 4n-1 (e.g. 3,7,11,15,19) this additionally consumes the null terminator
-                    hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1) ^ (ptr[0] | NormalizeToLowercase);
-                    hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ (ptr[1] | NormalizeToLowercase);
+                    hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1) ^ (p0 | NormalizeToLowercase);
+                    hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ (p1 | NormalizeToLowercase);
                     ptr += 2;
                 }
 
                 if (length > 0)
                 {
+                    uint p0 = ptr[0];
+                    if (!Utf16Utility.AllCharsInUInt32AreAscii(p0))
+                    {
+                        goto NotAscii;
+                    }
+
                     // Where length is 4n-3 (e.g. 1,5,9,13,17) this additionally consumes the null terminator
-                    hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ (ptr[0] | NormalizeToLowercase);
+                    hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ (p0 | NormalizeToLowercase);
+                }
+            }
+
+            return (int)(hash1 + (hash2 * 1566083941));
+
+        NotAscii:
+            return GetNonRandomizedHashCodeOrdinalIgnoreCaseSlow(this);
+
+            static int GetNonRandomizedHashCodeOrdinalIgnoreCaseSlow(string str)
+            {
+                int length = str.Length;
+                char[]? borrowedArr = null;
+                // Important: leave an additional space for '\0'
+                Span<char> scratch = (uint)length < 64 ?
+                    stackalloc char[64] : (borrowedArr = ArrayPool<char>.Shared.Rent(length + 1));
+
+                int charsWritten = System.Globalization.Ordinal.ToUpperOrdinal(str, scratch);
+                Debug.Assert(charsWritten == length);
+                scratch[length] = '\0';
+
+                const uint NormalizeToLowercase = 0x0020_0020u;
+                uint hash1 = (5381 << 16) + 5381;
+                uint hash2 = hash1;
+
+                // Duplicate the main loop, can be removed once JIT gets "Loop Unswitching" optimization
+                fixed (char* src = scratch)
+                {
+                    uint* ptr = (uint*)src;
+                    while (length > 2)
+                    {
+                        length -= 4;
+                        hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1) ^ (ptr[0] | NormalizeToLowercase);
+                        hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ (ptr[1] | NormalizeToLowercase);
+                        ptr += 2;
+                    }
+
+                    if (length > 0)
+                    {
+                        hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ (ptr[0] | NormalizeToLowercase);
+                    }
                 }
 
+                if (borrowedArr != null)
+                {
+                    ArrayPool<char>.Shared.Return(borrowedArr);
+                }
                 return (int)(hash1 + (hash2 * 1566083941));
             }
         }