Last 8 bytes of RSA modulus is keyid for V3 pubkeys (#205080)

- ported from rpm5.org work of Jeff Johnson
- minimal fix suitable for 4.4.2.x too, full rpm5.org changes require
  api changes which would be ok for HEAD but that's a bigger task...

diff --git a/rpmio/rpmpgp.c b/rpmio/rpmpgp.c
index 09e3fa8..1a4abe4 100644 (file)
--- a/rpmio/rpmpgp.c
+++ b/rpmio/rpmpgp.c
@@ -875,27 +875,39 @@ int pgpPrtComment(pgpTag tag, const uint8_t *h, size_t hlen)
     return 0;
 }
 
-int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen,
-               pgpKeyID_t keyid)
+int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen, pgpKeyID_t keyid)
 {
-    const uint8_t *s = pkt;
+    unsigned int val = *pkt;
+    size_t plen, hlen;
+    pgpTag tag;
+    const uint8_t *se, *h;
     DIGEST_CTX ctx;
-    uint8_t version;
     int rc = -1;       /* assume failure. */
 
-    if (pkt[0] != 0x99)
+    if (!(val & 0x80))
        return rc;
-    version = pkt[3];
 
-    switch (version) {
-    case 3:
-      {        pgpPktKeyV3 v = (pgpPktKeyV3) (pkt + 3);
+    if (val & 0x40) {
+       tag = (val & 0x3f);
+       plen = pgpLen(pkt+1, &hlen);
+    } else {
+       tag = (val >> 2) & 0xf;
+       plen = (1 << (val & 0x3));
+       hlen = pgpGrab(pkt+1, plen);
+    }
+    if (pktlen > 0 && 1 + plen + hlen > pktlen)
+       return rc;
+    
+    h = pkt + 1 + plen;
 
-       s += sizeof(pkt[0]) + sizeof(pkt[1]) + sizeof(pkt[2]) + sizeof(*v);
+    switch (h[0]) {
+    case 3:
+      {        pgpPktKeyV3 v = (pgpPktKeyV3) (h);
+       se = (uint8_t *)(v + 1);
        switch (v->pubkey_algo) {
        case PGPPUBKEYALGO_RSA:
-           s += (pgpMpiLen(s) - 8);
-           memmove(keyid, s, sizeof(keyid));
+           se += pgpMpiLen(se);
+           memmove(keyid, (se-8), 8);
            rc = 0;
            break;
        default:        /* TODO: md5 of mpi bodies (i.e. no length) */
@@ -903,31 +915,31 @@ int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen,
        }
       }        break;
     case 4:
-      {        pgpPktKeyV4 v = (pgpPktKeyV4) (pkt + 3);
-       uint8_t * SHA1 = NULL;
+      {        pgpPktKeyV4 v = (pgpPktKeyV4) (h);
+       uint8_t * d = NULL;
+       size_t dlen;
        int i;
 
-       s += sizeof(pkt[0]) + sizeof(pkt[1]) + sizeof(pkt[2]) + sizeof(*v);
+       se = (uint8_t *)(v + 1);
        switch (v->pubkey_algo) {
        case PGPPUBKEYALGO_RSA:
            for (i = 0; i < 2; i++)
-               s += pgpMpiLen(s);
+               se += pgpMpiLen(se);
            break;
        case PGPPUBKEYALGO_DSA:
            for (i = 0; i < 4; i++)
-               s += pgpMpiLen(s);
+               se += pgpMpiLen(se);
            break;
        }
 
        ctx = rpmDigestInit(PGPHASHALGO_SHA1, RPMDIGEST_NONE);
-       (void) rpmDigestUpdate(ctx, pkt, (s-pkt));
-       (void) rpmDigestFinal(ctx, (void **)&SHA1, NULL, 0);
+       (void) rpmDigestUpdate(ctx, pkt, (se-pkt));
+       (void) rpmDigestFinal(ctx, (void **)&d, &dlen, 0);
 
-       s = SHA1 + 12;
-       memmove(keyid, s, sizeof(keyid));
+       memmove(keyid, (d + (dlen-8)), 8);
+       if (d) free(d);
        rc = 0;
 
-       if (SHA1) free(SHA1);
       }        break;
     }
     return rc;