Fix ABI problem when marshalling X509VerifyStatusCode on s390x

The X509VerifyStatusCode type is defined as an enum in C code, but as
a struct (with a single member) in C# code.  This means that marshalling
this type only works correctly on platforms where the ABI treats these
two types as equivalent.  This fails e.g. on Linux on s390x.

Fixed by changing all native functions that take X509VerifyStatusCode
as argument or return type to use a plain "int" instead.

diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OCSP.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OCSP.cs
index f902743..edb731f 100644 (file)
--- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OCSP.cs
+++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OCSP.cs
@@ -35,14 +35,14 @@ internal static partial class Interop
         internal static extern void OcspResponseDestroy(IntPtr ocspReq);
 
         [DllImport(Libraries.CryptoNative)]
-        private static extern X509VerifyStatusCode CryptoNative_X509ChainGetCachedOcspStatus(
+        private static extern int CryptoNative_X509ChainGetCachedOcspStatus(
             SafeX509StoreCtxHandle ctx,
             string cachePath,
             int chainDepth);
 
         internal static X509VerifyStatusCode X509ChainGetCachedOcspStatus(SafeX509StoreCtxHandle ctx, string cachePath, int chainDepth)
         {
-            X509VerifyStatusCode response = CryptoNative_X509ChainGetCachedOcspStatus(ctx, cachePath, chainDepth);
+            X509VerifyStatusCode response = (X509VerifyStatusCode)CryptoNative_X509ChainGetCachedOcspStatus(ctx, cachePath, chainDepth);
 
             if (response.Code < 0)
             {
@@ -54,7 +54,7 @@ internal static partial class Interop
         }
 
         [DllImport(Libraries.CryptoNative)]
-        private static extern X509VerifyStatusCode CryptoNative_X509ChainVerifyOcsp(
+        private static extern int CryptoNative_X509ChainVerifyOcsp(
             SafeX509StoreCtxHandle ctx,
             SafeOcspRequestHandle req,
             SafeOcspResponseHandle resp,
@@ -68,7 +68,7 @@ internal static partial class Interop
             string cachePath,
             int chainDepth)
         {
-            X509VerifyStatusCode response = CryptoNative_X509ChainVerifyOcsp(ctx, req, resp, cachePath, chainDepth);
+            X509VerifyStatusCode response = (X509VerifyStatusCode)CryptoNative_X509ChainVerifyOcsp(ctx, req, resp, cachePath, chainDepth);
 
             if (response.Code < 0)
             {

diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.X509.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.X509.cs
index 885fe0e..0e5806d 100644 (file)
--- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.X509.cs
+++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.X509.cs
@@ -195,8 +195,13 @@ internal static partial class Interop
             return result != 0;
         }
 
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_X509StoreCtxGetError")]
-        internal static extern X509VerifyStatusCode X509StoreCtxGetError(SafeX509StoreCtxHandle ctx);
+        [DllImport(Libraries.CryptoNative)]
+        internal static extern int CryptoNative_X509StoreCtxGetError(SafeX509StoreCtxHandle ctx);
+
+        internal static X509VerifyStatusCode X509StoreCtxGetError(SafeX509StoreCtxHandle ctx)
+        {
+            return (X509VerifyStatusCode)CryptoNative_X509StoreCtxGetError(ctx);
+        }
 
         [DllImport(Libraries.CryptoNative)]
         private static extern int CryptoNative_X509StoreCtxReset(SafeX509StoreCtxHandle ctx);

diff --git a/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.c b/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.c
index 7bedeb3..54f1482 100644 (file)
--- a/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.c
+++ b/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.c
@@ -302,9 +302,9 @@ X509* CryptoNative_X509StoreCtxGetTargetCert(X509_STORE_CTX* ctx)
     return NULL;
 }
 
-X509VerifyStatusCode CryptoNative_X509StoreCtxGetError(X509_STORE_CTX* ctx)
+int32_t CryptoNative_X509StoreCtxGetError(X509_STORE_CTX* ctx)
 {
-    return (unsigned int)X509_STORE_CTX_get_error(ctx);
+    return (int32_t)X509_STORE_CTX_get_error(ctx);
 }
 
 int32_t CryptoNative_X509StoreCtxReset(X509_STORE_CTX* ctx)
@@ -337,7 +337,7 @@ int32_t CryptoNative_X509StoreCtxGetErrorDepth(X509_STORE_CTX* ctx)
     return X509_STORE_CTX_get_error_depth(ctx);
 }
 
-const char* CryptoNative_X509VerifyCertErrorString(X509VerifyStatusCode n)
+const char* CryptoNative_X509VerifyCertErrorString(int32_t n)
 {
     return X509_verify_cert_error_string((long)n);
 }
@@ -949,11 +949,11 @@ static time_t GetIssuanceWindowStart()
     return t;
 }
 
-X509VerifyStatusCode CryptoNative_X509ChainGetCachedOcspStatus(X509_STORE_CTX* storeCtx, char* cachePath, int chainDepth)
+int32_t CryptoNative_X509ChainGetCachedOcspStatus(X509_STORE_CTX* storeCtx, char* cachePath, int chainDepth)
 {
     if (storeCtx == NULL || cachePath == NULL)
     {
-        return (X509VerifyStatusCode)-1;
+        return -1;
     }
 
     X509* subject;
@@ -961,7 +961,7 @@ X509VerifyStatusCode CryptoNative_X509ChainGetCachedOcspStatus(X509_STORE_CTX* s
 
     if (!Get0CertAndIssuer(storeCtx, chainDepth, &subject, &issuer))
     {
-        return (X509VerifyStatusCode)-2;
+        return -2;
     }
 
     X509VerifyStatusCode ret = PAL_X509_V_ERR_UNABLE_TO_GET_CRL;
@@ -969,7 +969,7 @@ X509VerifyStatusCode CryptoNative_X509ChainGetCachedOcspStatus(X509_STORE_CTX* s
 
     if (fullPath == NULL)
     {
-        return ret;
+        return (int32_t)ret;
     }
 
     BIO* bio = BIO_new_file(fullPath, "rb");
@@ -1031,7 +1031,7 @@ X509VerifyStatusCode CryptoNative_X509ChainGetCachedOcspStatus(X509_STORE_CTX* s
         OCSP_RESPONSE_free(resp);
     }
 
-    return ret;
+    return (int32_t)ret;
 }
 
 OCSP_REQUEST* CryptoNative_X509ChainBuildOcspRequest(X509_STORE_CTX* storeCtx, int chainDepth)
@@ -1079,12 +1079,12 @@ OCSP_REQUEST* CryptoNative_X509ChainBuildOcspRequest(X509_STORE_CTX* storeCtx, i
     return req;
 }
 
-X509VerifyStatusCode
+int32_t
 CryptoNative_X509ChainVerifyOcsp(X509_STORE_CTX* storeCtx, OCSP_REQUEST* req, OCSP_RESPONSE* resp, char* cachePath, int chainDepth)
 {
     if (storeCtx == NULL || req == NULL || resp == NULL)
     {
-        return (X509VerifyStatusCode)-1;
+        return -1;
     }
 
     X509* subject;
@@ -1092,7 +1092,7 @@ CryptoNative_X509ChainVerifyOcsp(X509_STORE_CTX* storeCtx, OCSP_REQUEST* req, OC
 
     if (!Get0CertAndIssuer(storeCtx, chainDepth, &subject, &issuer))
     {
-        return (X509VerifyStatusCode)-2;
+        return -2;
     }
 
     X509VerifyStatusCode ret = PAL_X509_V_ERR_UNABLE_TO_GET_CRL;
@@ -1100,7 +1100,7 @@ CryptoNative_X509ChainVerifyOcsp(X509_STORE_CTX* storeCtx, OCSP_REQUEST* req, OC
 
     if (certId == NULL)
     {
-        return (X509VerifyStatusCode)-3;
+        return -3;
     }
 
     ASN1_GENERALIZEDTIME* thisUpdate = NULL;
@@ -1167,5 +1167,5 @@ CryptoNative_X509ChainVerifyOcsp(X509_STORE_CTX* storeCtx, OCSP_REQUEST* req, OC
         ASN1_GENERALIZEDTIME_free(thisUpdate);
     }
 
-    return ret;
+    return (int32_t)ret;
 }

diff --git a/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.h b/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.h
index d28f0cc..8b10534 100644 (file)
--- a/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.h
+++ b/src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.h
@@ -273,7 +273,7 @@ PALEXPORT X509* CryptoNative_X509StoreCtxGetTargetCert(X509_STORE_CTX* ctx);
 /*
 Shims the X509_STORE_CTX_get_error method.
 */
-PALEXPORT X509VerifyStatusCode CryptoNative_X509StoreCtxGetError(X509_STORE_CTX* ctx);
+PALEXPORT int32_t CryptoNative_X509StoreCtxGetError(X509_STORE_CTX* ctx);
 
 /*
 Resets ctx to before the chain was built, preserving the target cert, trust store, extra cert context,
@@ -301,7 +301,7 @@ PALEXPORT void CryptoNative_X509StoreCtxSetVerifyCallback(X509_STORE_CTX* ctx, X
 /*
 Shims the X509_verify_cert_error_string method.
 */
-PALEXPORT const char* CryptoNative_X509VerifyCertErrorString(X509VerifyStatusCode n);
+PALEXPORT const char* CryptoNative_X509VerifyCertErrorString(int32_t n);
 
 /*
 Shims the X509_CRL_free method.
@@ -378,7 +378,7 @@ PALEXPORT int32_t CryptoNative_X509StoreCtxResetForSignatureError(X509_STORE_CTX
 Look for a cached OCSP response appropriate to the end-entity certificate using the issuer as
 determined by the chain in storeCtx.
 */
-PALEXPORT X509VerifyStatusCode CryptoNative_X509ChainGetCachedOcspStatus(X509_STORE_CTX* storeCtx, char* cachePath, int chainDepth);
+PALEXPORT int32_t CryptoNative_X509ChainGetCachedOcspStatus(X509_STORE_CTX* storeCtx, char* cachePath, int chainDepth);
 
 /*
 Build an OCSP request appropriate for the end-entity certificate using the issuer (and trust) as
@@ -390,8 +390,8 @@ PALEXPORT OCSP_REQUEST* CryptoNative_X509ChainBuildOcspRequest(X509_STORE_CTX* s
 Determine if the OCSP response is acceptable, and if acceptable report the status and
 cache the result (if appropriate)
 */
-PALEXPORT X509VerifyStatusCode CryptoNative_X509ChainVerifyOcsp(X509_STORE_CTX* storeCtx,
-                                                                OCSP_REQUEST* req,
-                                                                OCSP_RESPONSE* resp,
-                                                                char* cachePath,
-                                                                int chainDepth);
+PALEXPORT int32_t CryptoNative_X509ChainVerifyOcsp(X509_STORE_CTX* storeCtx,
+                                                   OCSP_REQUEST* req,
+                                                   OCSP_RESPONSE* resp,
+                                                   char* cachePath,
+                                                   int chainDepth);