Fix Running .NET Core Under Windows App Compat Shim (dotnet/coreclr#26605)

* Fix Running .NET Core Under Winndows App Compat Shim

.NET Core uses Windows features called sort handles for perf reason to speed up the collation operations (e.g. string comparisons). When enabling Windows app compat shim (e.g. Windows 7 compatibility mode), the calls uses sort handles will fail which cause all collation operations to fail.
The fix here is to detect if the sort handles work before using it to ensure successful run.

* Address the feedback

* More feedback addressing

Commit migrated from https://github.com/dotnet/coreclr/commit/0f1314bdca91add2b4e9dbc95ec687dab638e1a0

diff --git a/src/libraries/System.Private.CoreLib/src/Interop/Windows/Kernel32/Interop.Globalization.cs b/src/libraries/System.Private.CoreLib/src/Interop/Windows/Kernel32/Interop.Globalization.cs
index 3115466..c83bd35 100644 (file)
--- a/src/libraries/System.Private.CoreLib/src/Interop/Windows/Kernel32/Interop.Globalization.cs
+++ b/src/libraries/System.Private.CoreLib/src/Interop/Windows/Kernel32/Interop.Globalization.cs
@@ -24,6 +24,9 @@ internal static partial class Interop
         internal const uint LOCALE_RETURN_NUMBER        = 0x20000000;
         internal const uint LOCALE_NOUSEROVERRIDE       = 0x80000000;
 
+        internal const uint LCMAP_SORTHANDLE            = 0x20000000;
+        internal const uint LCMAP_HASH                  = 0x00040000;
+
         internal const int  COMPARE_STRING              = 0x0001;
 
         internal const uint TIME_NOSECONDS = 0x00000002;

diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.Windows.cs
index d9bbc89..67f4982 100644 (file)
--- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.Windows.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.Windows.cs
@@ -10,22 +10,36 @@ namespace System.Globalization
 {
     public partial class CompareInfo
     {
-        private unsafe void InitSort(CultureInfo culture)
+        internal static unsafe IntPtr GetSortHandle(string cultureName)
         {
-            _sortName = culture.SortName;
-
             if (GlobalizationMode.Invariant)
             {
-                _sortHandle = IntPtr.Zero;
+                return IntPtr.Zero;
             }
-            else
-            {
-                const uint LCMAP_SORTHANDLE = 0x20000000;
 
-                IntPtr handle;
-                int ret = Interop.Kernel32.LCMapStringEx(_sortName, LCMAP_SORTHANDLE, null, 0, &handle, IntPtr.Size, null, null, IntPtr.Zero);
-                _sortHandle = ret > 0 ? handle : IntPtr.Zero;
+            IntPtr handle;
+            int ret = Interop.Kernel32.LCMapStringEx(cultureName, Interop.Kernel32.LCMAP_SORTHANDLE, null, 0, &handle, IntPtr.Size, null, null, IntPtr.Zero);
+            if (ret > 0)
+            {
+                // Even if we can get the sort handle, it is not guaranteed to work when Windows compatibility shim is applied
+                // e.g. Windows 7 compatibility mode. We need to ensure it is working before using it.
+                // otherwise the whole framework app will not start.
+                int hashValue = 0;
+                char a = 'a';
+                ret = Interop.Kernel32.LCMapStringEx(null, Interop.Kernel32.LCMAP_HASH, &a, 1, &hashValue, sizeof(int), null, null, handle);
+                if (ret > 1)
+                {
+                    return handle;
+                }
             }
+
+            return IntPtr.Zero;
+        }
+
+        private void InitSort(CultureInfo culture)
+        {
+            _sortName = culture.SortName;
+            _sortHandle = GetSortHandle(_sortName);
         }
 
         private static unsafe int FindStringOrdinal(

diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.Windows.cs
index d0c8594..9486447 100644 (file)
--- a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.Windows.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.Windows.cs
@@ -10,17 +10,7 @@ namespace System.Globalization
     {
         private unsafe void FinishInitialization()
         {
-            if (GlobalizationMode.Invariant)
-            {
-                _sortHandle = IntPtr.Zero;
-                return;
-            }
-
-            const uint LCMAP_SORTHANDLE = 0x20000000;
-
-            IntPtr handle;
-            int ret = Interop.Kernel32.LCMapStringEx(_textInfoName, LCMAP_SORTHANDLE, null, 0, &handle, IntPtr.Size, null, null, IntPtr.Zero);
-            _sortHandle = ret > 0 ? handle : IntPtr.Zero;
+            _sortHandle = CompareInfo.GetSortHandle(_textInfoName);
         }
 
         private unsafe void ChangeCase(char* pSource, int pSourceLen, char* pResult, int pResultLen, bool toUpper)