Get rid of long since deprecated VFY_VerifyDigest() uses

- VFY_VerifyDigest() has been deprecated since NSS >= 3.12 and for
  a good reason too: with VFY_VerifyDigest() caller needs to painfully
  enumerate every possible supported enc + hash combination, only for
  NSS to revert the process. Use the saner VFY_VerifyDigestDirect()
  interface instead and test for its presence in configure.
- This means we now require NSS >= 3.12 but as that's already 4.5 years
  old and included in ancient beasts like RHEL-4, this doesn't seem
  exactly unreasonable requirement. And then there's always beecrypt...
(cherry picked from commit 9b995a7674adba08248fac79ae8b23ecbecc13de)

diff --git a/INSTALL b/INSTALL
index 96ddbdb..85cc8d6 100644 (file)
--- a/INSTALL
+++ b/INSTALL
@@ -9,7 +9,7 @@ The libmagic (aka file) library for file type detection (used by rpmbuild).
 The source for the file utility + library is available from
     ftp://ftp.astron.com/pub/file/
 
-The NSS library for encryption, and NSPR library which NSS uses. 
+The NSS >= 3.12 library for encryption, and NSPR library which NSS uses. 
 Both NSPR and NSS libraries and headers need to be installed during RPM
 compilation. As NSPR and NSS typically install their headers outside
 the regular include search path, you need to tell configure about this,

diff --git a/configure.ac b/configure.ac
index 1578cdb..a10a028 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -283,14 +283,14 @@ if test "$with_beecrypt" != yes ; then
 AC_CHECK_HEADERS([nspr.h nss.h sechash.h], [], [
   AC_MSG_ERROR([missing required NSPR / NSS header])
 ])
-AC_CHECK_LIB(nss3, NSS_NoDB_Init, [
+AC_CHECK_LIB(nss3, VFY_VerifyDigestDirect, [
   WITH_NSS_LIB=-lnss3
   AC_CHECK_LIB(nss3, NSS_InitContext, [
     AC_DEFINE(HAVE_NSS_INITCONTEXT, 1, [Define to 1 if NSS has NSS_InitContext])
     AC_SUBST(HAVE_NSS_INITCONTEXT, [1])
   ])
 ], [
-  AC_MSG_ERROR([missing required NSS library 'nss3'])
+  AC_MSG_ERROR([required NSS library 'nss3' missing or too old])
 ])
 fi
 AC_SUBST(WITH_NSS_INCLUDE)

diff --git a/rpmio/digest_nss.c b/rpmio/digest_nss.c
index f1e5d6a..f3ab57f 100644 (file)
--- a/rpmio/digest_nss.c
+++ b/rpmio/digest_nss.c
@@ -220,6 +220,21 @@ int rpmDigestFinal(DIGEST_CTX ctx, void ** datap, size_t *lenp, int asAscii)
     return 0;
 }
 
+RPM_GNUC_PURE
+static SECOidTag getHashAlg(unsigned int hashalgo)
+{
+    switch (hashalgo) {
+    case PGPHASHALGO_MD5:      return SEC_OID_MD5;
+    case PGPHASHALGO_MD2:      return SEC_OID_MD2;
+    case PGPHASHALGO_SHA1:     return SEC_OID_SHA1;
+    case PGPHASHALGO_SHA224:   return SEC_OID_SHA224;
+    case PGPHASHALGO_SHA256:   return SEC_OID_SHA256;
+    case PGPHASHALGO_SHA384:   return SEC_OID_SHA384;
+    case PGPHASHALGO_SHA512:   return SEC_OID_SHA512;
+    }
+    return SEC_OID_UNKNOWN;
+}
+
 static int pgpMpiSet(unsigned int lbits, uint8_t *dest,
                     const uint8_t * p, const uint8_t * pend)
 {
@@ -370,11 +385,15 @@ static int pgpVerifySigDSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig,
                           uint8_t *hash, size_t hashlen, int hash_algo)
 {
     SECItem digest = { .type = siBuffer, .data = hash, .len = hashlen };
-    SECOidTag sigalg = SEC_OID_ANSIX9_DSA_SIGNATURE_WITH_SHA1_DIGEST;
+    SECOidTag encAlg = SEC_OID_ANSIX9_DSA_SIGNATURE;
+    SECOidTag hashAlg = getHashAlg(hash_algo);
     SECStatus rc;
 
-    /* XXX VFY_VerifyDigest() is deprecated in NSS 3.12 */ 
-    rc = VFY_VerifyDigest(&digest, pgpkey->data, pgpsig->data, sigalg, NULL);
+    if (hashAlg == SEC_OID_UNKNOWN)
+       return 1;
+
+    rc = VFY_VerifyDigestDirect(&digest, pgpkey->data, pgpsig->data,
+                               encAlg, hashAlg, NULL);
 
     return (rc != SECSuccess);
 }
@@ -422,33 +441,13 @@ static int pgpVerifySigRSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig,
     SECItem *sig = pgpsig->data;
     SECKEYPublicKey *key = pgpkey->data;
     SECItem *padded = NULL;
-    SECOidTag sigalg;
+    SECOidTag encAlg = SEC_OID_PKCS1_RSA_ENCRYPTION;
+    SECOidTag hashAlg = getHashAlg(hash_algo);
     SECStatus rc = SECFailure;
     size_t siglen, padlen;
 
-    switch (hash_algo) {
-    case PGPHASHALGO_MD5:
-       sigalg = SEC_OID_PKCS1_MD5_WITH_RSA_ENCRYPTION;
-       break;
-    case PGPHASHALGO_MD2:
-       sigalg = SEC_OID_PKCS1_MD2_WITH_RSA_ENCRYPTION;
-       break;
-    case PGPHASHALGO_SHA1:
-       sigalg = SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION;
-       break;
-    case PGPHASHALGO_SHA256:
-       sigalg = SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION;
-       break;
-    case PGPHASHALGO_SHA384:
-        sigalg = SEC_OID_PKCS1_SHA384_WITH_RSA_ENCRYPTION;
-       break;
-    case PGPHASHALGO_SHA512:
-       sigalg = SEC_OID_PKCS1_SHA512_WITH_RSA_ENCRYPTION;
-       break;
-    default:
-       return 1; /* dont bother with unknown hash types */
-       break;
-    }
+    if (hashAlg == SEC_OID_UNKNOWN)
+       return 1;
 
     /* Zero-pad signature to expected size if necessary */
     siglen = SECKEY_SignatureLen(key);
@@ -462,8 +461,7 @@ static int pgpVerifySigRSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig,
        sig = padded;
     }
 
-    /* XXX VFY_VerifyDigest() is deprecated in NSS 3.12 */ 
-    rc = VFY_VerifyDigest(&digest, key, sig, sigalg, NULL);
+    rc = VFY_VerifyDigestDirect(&digest, key, sig, encAlg, hashAlg, NULL);
 
     if (padded)
        SECITEM_ZfreeItem(padded, PR_TRUE);