Avoid failures when allocate memory in native (#8028)

* Avoid failures when allocate memory in native

using new in the native side can throw and the app will shutdown. instead we detect the failure and throw a managed exception

* Use same mechanism we return the error code from GetSortHandle

* Add assert

* Dispose the safeHandle

diff --git a/src/corefx/System.Globalization.Native/collation.cpp b/src/corefx/System.Globalization.Native/collation.cpp
index 06d41f9..f372112 100644 (file)
--- a/src/corefx/System.Globalization.Native/collation.cpp
+++ b/src/corefx/System.Globalization.Native/collation.cpp
@@ -10,6 +10,8 @@
 #include <map>
 
 #include "icushim.h"
+#include "locale.hpp"
+#include "errors.h"
 
 const int32_t CompareOptionsIgnoreCase = 0x1;
 const int32_t CompareOptionsIgnoreNonSpace = 0x2;
@@ -331,24 +333,30 @@ extern "C" int32_t GlobalizationNative_GetSortVersion()
     return UCOL_RUNTIME_VERSION << 16 | UCOL_BUILDER_VERSION;
 }
 
-extern "C" SortHandle* GlobalizationNative_GetSortHandle(const char* lpLocaleName)
+extern "C" ResultCode GlobalizationNative_GetSortHandle(const char* lpLocaleName, SortHandle** ppSortHandle)
 {
-    SortHandle* pSortHandle = new SortHandle();
+    assert(ppSortHandle != nullptr);
+    
+    *ppSortHandle = new (std::nothrow) SortHandle();
+    if ((*ppSortHandle) == nullptr)
+    {
+        return GetResultCode(U_MEMORY_ALLOCATION_ERROR);
+    }
 
     UErrorCode err = U_ZERO_ERROR;
 
-    pSortHandle->regular = ucol_open(lpLocaleName, &err);
+    (*ppSortHandle)->regular = ucol_open(lpLocaleName, &err);
 
     if (U_FAILURE(err))
     {
-        if (pSortHandle->regular != nullptr)
-              ucol_close(pSortHandle->regular);
+        if ((*ppSortHandle)->regular != nullptr)
+            ucol_close((*ppSortHandle)->regular);
 
-        delete pSortHandle;
-        pSortHandle = nullptr;
+        delete (*ppSortHandle);
+        (*ppSortHandle) = nullptr;
     }
 
-    return pSortHandle;
+    return GetResultCode(err);
 }
 
 extern "C" void GlobalizationNative_CloseSortHandle(SortHandle* pSortHandle)

diff --git a/src/corefx/System.Globalization.Native/errors.h b/src/corefx/System.Globalization.Native/errors.h
index 031ea7b..b23a0da 100644 (file)
--- a/src/corefx/System.Globalization.Native/errors.h
+++ b/src/corefx/System.Globalization.Native/errors.h
@@ -13,6 +13,7 @@ enum ResultCode : int32_t
     Success = 0,
     UnknownError = 1,
     InsufficentBuffer = 2,
+    OutOfMemory = 3
 };
 
 /*
@@ -24,6 +25,11 @@ static ResultCode GetResultCode(UErrorCode err)
     {
         return InsufficentBuffer;
     }
+    
+    if (err == U_MEMORY_ALLOCATION_ERROR)
+    {
+        return OutOfMemory;
+    }
 
     if (U_SUCCESS(err))
     {

diff --git a/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.Collation.cs b/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.Collation.cs
index 111a782..eb9e074 100644 (file)
--- a/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.Collation.cs
+++ b/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.Collation.cs
@@ -13,7 +13,7 @@ internal static partial class Interop
     {
         [SecurityCritical]
         [DllImport(Libraries.GlobalizationInterop, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_GetSortHandle")]
-        internal unsafe static extern SafeSortHandle GetSortHandle(byte[] localeName);
+        internal unsafe static extern ResultCode GetSortHandle(byte[] localeName, out SafeSortHandle sortHandle);
 
         [SecurityCritical]
         [DllImport(Libraries.GlobalizationInterop, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_CloseSortHandle")]

diff --git a/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.ResultCode.cs b/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.ResultCode.cs
index 4621580..cca6ae4 100644 (file)
--- a/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.ResultCode.cs
+++ b/src/mscorlib/corefx/Interop/Unix/System.Globalization.Native/Interop.ResultCode.cs
@@ -12,6 +12,7 @@ internal static partial class Interop
             Success = 0,
             UnknownError = 1,
             InsufficentBuffer = 2,
+            OutOfMemory = 3
         }
     }
 }

diff --git a/src/mscorlib/corefx/SR.cs b/src/mscorlib/corefx/SR.cs
index b14642c..e963e47 100644 (file)
--- a/src/mscorlib/corefx/SR.cs
+++ b/src/mscorlib/corefx/SR.cs
@@ -11,6 +11,11 @@ internal static class SR
         get { return Environment.GetResourceString("Arg_ArrayZeroError"); }
     }
 
+    public static string Arg_ExternalException
+    {
+        get { return Environment.GetResourceString("Arg_ExternalException"); }
+    }
+
     public static string Arg_HexStyleNotSupported
     {
         get { return Environment.GetResourceString("Arg_HexStyleNotSupported"); }

diff --git a/src/mscorlib/corefx/System/Globalization/CompareInfo.Unix.cs b/src/mscorlib/corefx/System/Globalization/CompareInfo.Unix.cs
index 608c35f..01483b2 100644 (file)
--- a/src/mscorlib/corefx/System/Globalization/CompareInfo.Unix.cs
+++ b/src/mscorlib/corefx/System/Globalization/CompareInfo.Unix.cs
@@ -26,7 +26,16 @@ namespace System.Globalization
         private void InitSort(CultureInfo culture)
         {
             _sortName = culture.SortName;
-            _sortHandle = Interop.GlobalizationInterop.GetSortHandle(GetNullTerminatedUtf8String(_sortName));
+            Interop.GlobalizationInterop.ResultCode resultCode = Interop.GlobalizationInterop.GetSortHandle(GetNullTerminatedUtf8String(_sortName), out _sortHandle); 
+            if (resultCode != Interop.GlobalizationInterop.ResultCode.Success)
+            {
+                _sortHandle.Dispose();
+                
+                if (resultCode == Interop.GlobalizationInterop.ResultCode.OutOfMemory)
+                    throw new OutOfMemoryException();
+                
+                throw new ExternalException(SR.Arg_ExternalException);
+            }
             _isAsciiEqualityOrdinal = (_sortName == "en-US" || _sortName == "");
         }