Use checked arithmetic in Pkcs12Kdf

When calculating the length of P and I, it's possible for this to result in an arithmetic overflow.
This arithmetic overflow is then fed in to `stackalloc`, which treats the input as unsigned and
causes a large stack allocation.

diff --git a/src/libraries/Common/src/System/Security/Cryptography/Pkcs12Kdf.cs b/src/libraries/Common/src/System/Security/Cryptography/Pkcs12Kdf.cs
index 132b7a7..73f07e3 100644 (file)
--- a/src/libraries/Common/src/System/Security/Cryptography/Pkcs12Kdf.cs
+++ b/src/libraries/Common/src/System/Security/Cryptography/Pkcs12Kdf.cs
@@ -130,10 +130,10 @@ namespace System.Security.Cryptography.Pkcs
             // (The RFC quote considers the trailing '\0' to be part of the string,
             // so "empty string" from this RFC means "null string" in C#, and C#'s
             // "empty string" is not 'empty' in this context.)
-            int PLen = ((passLen - 1 + vBytes) / vBytes) * vBytes;
+            int PLen = checked(((passLen - 1 + vBytes) / vBytes) * vBytes);
 
             // 4.  Set I=S||P to be the concatenation of S and P.
-            int ILen = SLen + PLen;
+            int ILen = checked(SLen + PLen);
             Span<byte> I = stackalloc byte[0];
             byte[]? IRented = null;