Exploration: Retry STATUS_UNSUCCESSFUL from CNG (dotnet/corefx#41300)

* Exploration: Retry STATUS_UNSUCCESSFUL from CNG

* Restore Array.Resize to separate changes

Commit migrated from https://github.com/dotnet/corefx/commit/ac99a1b7168bd32046a954c3f06012c0fa909bed

diff --git a/src/libraries/Common/src/Internal/Cryptography/CngCommon.SignVerify.cs b/src/libraries/Common/src/Internal/Cryptography/CngCommon.SignVerify.cs
index 6f19f69..c3da139 100644 (file)
--- a/src/libraries/Common/src/Internal/Cryptography/CngCommon.SignVerify.cs
+++ b/src/libraries/Common/src/Internal/Cryptography/CngCommon.SignVerify.cs
@@ -11,6 +11,8 @@ namespace Internal.Cryptography
 {
     internal static partial class CngCommon
     {
+        private const int StatusUnsuccessfulRetryCount = 1;
+
         public static unsafe byte[] SignHash(this SafeNCryptKeyHandle keyHandle, ReadOnlySpan<byte> hash, AsymmetricPaddingMode paddingMode, void* pPaddingInfo, int estimatedSize)
         {
 #if DEBUG
@@ -19,11 +21,23 @@ namespace Internal.Cryptography
             byte[] signature = new byte[estimatedSize];
             int numBytesNeeded;
             ErrorCode errorCode = Interop.NCrypt.NCryptSignHash(keyHandle, pPaddingInfo, hash, hash.Length, signature, signature.Length, out numBytesNeeded, paddingMode);
+
+            if (errorCode == ErrorCode.STATUS_UNSUCCESSFUL)
+            {
+                errorCode = Interop.NCrypt.NCryptSignHash(keyHandle, pPaddingInfo, hash, hash.Length, signature, signature.Length, out numBytesNeeded, paddingMode);
+            }
+
             if (errorCode == ErrorCode.NTE_BUFFER_TOO_SMALL)
             {
                 signature = new byte[numBytesNeeded];
                 errorCode = Interop.NCrypt.NCryptSignHash(keyHandle, pPaddingInfo, hash, hash.Length, signature, signature.Length, out numBytesNeeded, paddingMode);
             }
+
+            if (errorCode == ErrorCode.STATUS_UNSUCCESSFUL)
+            {
+                errorCode = Interop.NCrypt.NCryptSignHash(keyHandle, pPaddingInfo, hash, hash.Length, signature, signature.Length, out numBytesNeeded, paddingMode);
+            }
+
             if (errorCode != ErrorCode.ERROR_SUCCESS)
                 throw errorCode.ToCryptographicException();
 
@@ -33,25 +47,49 @@ namespace Internal.Cryptography
 
         public static unsafe bool TrySignHash(this SafeNCryptKeyHandle keyHandle, ReadOnlySpan<byte> hash, Span<byte> signature, AsymmetricPaddingMode paddingMode, void* pPaddingInfo, out int bytesWritten)
         {
-            ErrorCode error = Interop.NCrypt.NCryptSignHash(keyHandle, pPaddingInfo, hash, hash.Length, signature, signature.Length, out int numBytesNeeded, paddingMode);
-            switch (error)
+            for (int i = 0; i <= StatusUnsuccessfulRetryCount; i++)
             {
-                case ErrorCode.ERROR_SUCCESS:
-                    bytesWritten = numBytesNeeded;
-                    return true;
+                ErrorCode error = Interop.NCrypt.NCryptSignHash(
+                    keyHandle,
+                    pPaddingInfo,
+                    hash,
+                    hash.Length,
+                    signature,
+                    signature.Length,
+                    out int numBytesNeeded,
+                    paddingMode);
+
+                switch (error)
+                {
+                    case ErrorCode.ERROR_SUCCESS:
+                        bytesWritten = numBytesNeeded;
+                        return true;
 
-                case ErrorCode.NTE_BUFFER_TOO_SMALL:
-                    bytesWritten = 0;
-                    return false;
+                    case ErrorCode.NTE_BUFFER_TOO_SMALL:
+                        bytesWritten = 0;
+                        return false;
 
-                default:
-                    throw error.ToCryptographicException();
+                    case ErrorCode.STATUS_UNSUCCESSFUL:
+                        // Retry
+                        break;
+
+                    default:
+                        throw error.ToCryptographicException();
+                }
             }
+
+            throw ErrorCode.STATUS_UNSUCCESSFUL.ToCryptographicException();
         }
 
         public static unsafe bool VerifyHash(this SafeNCryptKeyHandle keyHandle, ReadOnlySpan<byte> hash, ReadOnlySpan<byte> signature, AsymmetricPaddingMode paddingMode, void* pPaddingInfo)
         {
             ErrorCode errorCode = Interop.NCrypt.NCryptVerifySignature(keyHandle, pPaddingInfo, hash, hash.Length, signature, signature.Length, paddingMode);
+
+            if (errorCode == ErrorCode.STATUS_UNSUCCESSFUL)
+            {
+                errorCode = Interop.NCrypt.NCryptVerifySignature(keyHandle, pPaddingInfo, hash, hash.Length, signature, signature.Length, paddingMode);
+            }
+
             return errorCode == ErrorCode.ERROR_SUCCESS;  // For consistency with other AsymmetricAlgorithm-derived classes, return "false" for any error code rather than making the caller catch an exception.
         }
     }

diff --git a/src/libraries/Common/src/Interop/Windows/NCrypt/Interop.ErrorCode.cs b/src/libraries/Common/src/Interop/Windows/NCrypt/Interop.ErrorCode.cs
index 9cc05dc..14bdb60 100644 (file)
--- a/src/libraries/Common/src/Interop/Windows/NCrypt/Interop.ErrorCode.cs
+++ b/src/libraries/Common/src/Interop/Windows/NCrypt/Interop.ErrorCode.cs
@@ -20,6 +20,7 @@ internal static partial class Interop
             NTE_NOT_SUPPORTED = unchecked((int)0x80090029),
             NTE_NO_MORE_ITEMS = unchecked((int)0x8009002a),
             E_FAIL = unchecked((int)0x80004005),
+            STATUS_UNSUCCESSFUL = unchecked((int)0xC0000001),
         }
     }
 }

diff --git a/src/libraries/Common/src/System/Security/Cryptography/RSACng.EncryptDecrypt.cs b/src/libraries/Common/src/System/Security/Cryptography/RSACng.EncryptDecrypt.cs
index 9874132..66b636c 100644 (file)
--- a/src/libraries/Common/src/System/Security/Cryptography/RSACng.EncryptDecrypt.cs
+++ b/src/libraries/Common/src/System/Security/Cryptography/RSACng.EncryptDecrypt.cs
@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System.Buffers;
 using System.Runtime.InteropServices;
 using Microsoft.Win32.SafeHandles;
 using Internal.Cryptography;
@@ -20,6 +19,7 @@ namespace System.Security.Cryptography
     public sealed partial class RSACng : RSA
     {
         private const int Pkcs1PaddingOverhead = 11;
+        private const int StatusUnsuccessfulRetryCount = 1;
 
         /// <summary>Encrypts data using the public key.</summary>
         public override unsafe byte[] Encrypt(byte[] data, RSAEncryptionPadding padding) =>
@@ -224,19 +224,37 @@ namespace System.Security.Cryptography
 #endif
 
             byte[] output = new byte[estimatedSize];
-            int numBytesNeeded;
-            ErrorCode errorCode = encrypt ?
-                Interop.NCrypt.NCryptEncrypt(key, input, input.Length, paddingInfo, output, output.Length, out numBytesNeeded, paddingMode) :
-                Interop.NCrypt.NCryptDecrypt(key, input, input.Length, paddingInfo, output, output.Length, out numBytesNeeded, paddingMode);
+            int numBytesNeeded = 0;
+
+            ErrorCode errorCode = 0;
+
+            for (int i = 0; i <= StatusUnsuccessfulRetryCount; i++)
+            {
+                errorCode =
+                    EncryptOrDecrypt(key, input, output, paddingMode, paddingInfo, encrypt, out numBytesNeeded);
+
+                if (errorCode != ErrorCode.STATUS_UNSUCCESSFUL)
+                {
+                    break;
+                }
+            }
 
             if (errorCode == ErrorCode.NTE_BUFFER_TOO_SMALL)
             {
                 output = new byte[numBytesNeeded];
-                errorCode = encrypt ?
-                    Interop.NCrypt.NCryptEncrypt(key, input, input.Length, paddingInfo, output, output.Length, out numBytesNeeded, paddingMode) :
-                    Interop.NCrypt.NCryptDecrypt(key, input, input.Length, paddingInfo, output, output.Length, out numBytesNeeded, paddingMode);
 
+                for (int i = 0; i <= StatusUnsuccessfulRetryCount; i++)
+                {
+                    errorCode =
+                        EncryptOrDecrypt(key, input, output, paddingMode, paddingInfo, encrypt, out numBytesNeeded);
+
+                    if (errorCode != ErrorCode.STATUS_UNSUCCESSFUL)
+                    {
+                        break;
+                    }
+                }
             }
+
             if (errorCode != ErrorCode.ERROR_SUCCESS)
             {
                 throw errorCode.ToCryptographicException();
@@ -249,22 +267,42 @@ namespace System.Security.Cryptography
         // Now that the padding mode and information have been marshaled to their native counterparts, perform the encryption or decryption.
         private unsafe bool TryEncryptOrDecrypt(SafeNCryptKeyHandle key, ReadOnlySpan<byte> input, Span<byte> output, AsymmetricPaddingMode paddingMode, void* paddingInfo, bool encrypt, out int bytesWritten)
         {
-            int numBytesNeeded;
-            ErrorCode errorCode = encrypt ?
-                Interop.NCrypt.NCryptEncrypt(key, input, input.Length, paddingInfo, output, output.Length, out numBytesNeeded, paddingMode) :
-                Interop.NCrypt.NCryptDecrypt(key, input, input.Length, paddingInfo, output, output.Length, out numBytesNeeded, paddingMode);
-
-            switch (errorCode)
+            for (int i = 0; i <= StatusUnsuccessfulRetryCount; i++)
             {
-                case ErrorCode.ERROR_SUCCESS:
-                    bytesWritten = numBytesNeeded;
-                    return true;
-                case ErrorCode.NTE_BUFFER_TOO_SMALL:
-                    bytesWritten = 0;
-                    return false;
-                default:
-                    throw errorCode.ToCryptographicException();
+                int numBytesNeeded;
+                ErrorCode errorCode =
+                    EncryptOrDecrypt(key, input, output, paddingMode, paddingInfo, encrypt, out numBytesNeeded);
+
+                switch (errorCode)
+                {
+                    case ErrorCode.ERROR_SUCCESS:
+                        bytesWritten = numBytesNeeded;
+                        return true;
+                    case ErrorCode.NTE_BUFFER_TOO_SMALL:
+                        bytesWritten = 0;
+                        return false;
+                    case ErrorCode.STATUS_UNSUCCESSFUL:
+                        break;
+                    default:
+                        throw errorCode.ToCryptographicException();
+                }
             }
+
+            throw ErrorCode.STATUS_UNSUCCESSFUL.ToCryptographicException();
+        }
+
+        private static unsafe ErrorCode EncryptOrDecrypt(
+            SafeNCryptKeyHandle key,
+            ReadOnlySpan<byte> input,
+            Span<byte> output,
+            AsymmetricPaddingMode paddingMode,
+            void* paddingInfo,
+            bool encrypt,
+            out int bytesNeeded)
+        {
+            return encrypt ?
+                Interop.NCrypt.NCryptEncrypt(key, input, input.Length, paddingInfo, output, output.Length, out bytesNeeded, paddingMode) :
+                Interop.NCrypt.NCryptDecrypt(key, input, input.Length, paddingInfo, output, output.Length, out bytesNeeded, paddingMode);
         }
     }
 #if INTERNAL_ASYMMETRIC_IMPLEMENTATIONS