src/openssl/kw_des.c

   1 /**
   2  *
   3  * XMLSec library
   4  *
   5  * DES Algorithm support
   6  *
   7  * This is free software; see Copyright file in the source
   8  * distribution for preciese wording.
   9  *
  10  * Copyright (C) 2002-2003 Aleksey Sanin <aleksey@aleksey.com>
  11  */
  12 #ifndef XMLSEC_NO_DES
  13 #include "globals.h"
  14
  15 #include <stdlib.h>
  16 #include <stdio.h>
  17 #include <string.h>
  18
  19 #include <openssl/des.h>
  20 #include <openssl/rand.h>
  21 #include <openssl/sha.h>
  22
  23 #include <xmlsec/xmlsec.h>
  24 #include <xmlsec/xmltree.h>
  25 #include <xmlsec/keys.h>
  26 #include <xmlsec/transforms.h>
  27 #include <xmlsec/errors.h>
  28
  29 #include <xmlsec/openssl/crypto.h>
  30
  31 #define XMLSEC_OPENSSL_DES3_KEY_LENGTH                          24
  32 #define XMLSEC_OPENSSL_DES3_IV_LENGTH                           8
  33 #define XMLSEC_OPENSSL_DES3_BLOCK_LENGTH                        8
  34
  35 /*********************************************************************
  36  *
  37  * Triple DES Key Wrap transform
  38  *
  39  * key (xmlSecBuffer) is located after xmlSecTransform structure
  40  *
  41  ********************************************************************/
  42 #define xmlSecOpenSSLKWDes3GetKey(transform) \
  43     ((xmlSecBufferPtr)(((xmlSecByte*)(transform)) + sizeof(xmlSecTransform)))
  44 #define xmlSecOpenSSLKWDes3Size \
  45     (sizeof(xmlSecTransform) + sizeof(xmlSecBuffer))
  46
  47 static int      xmlSecOpenSSLKWDes3Initialize                   (xmlSecTransformPtr transform);
  48 static void     xmlSecOpenSSLKWDes3Finalize                     (xmlSecTransformPtr transform);
  49 static int      xmlSecOpenSSLKWDes3SetKeyReq                    (xmlSecTransformPtr transform,
  50                                                                  xmlSecKeyReqPtr keyReq);
  51 static int      xmlSecOpenSSLKWDes3SetKey                       (xmlSecTransformPtr transform,
  52                                                                  xmlSecKeyPtr key);
  53 static int      xmlSecOpenSSLKWDes3Execute                      (xmlSecTransformPtr transform,
  54                                                                  int last,
  55                                                                  xmlSecTransformCtxPtr transformCtx);
  56 static int      xmlSecOpenSSLKWDes3Encode                       (const xmlSecByte *key,
  57                                                                  xmlSecSize keySize,
  58                                                                  const xmlSecByte *in,
  59                                                                  xmlSecSize inSize,
  60                                                                  xmlSecByte *out,
  61                                                                  xmlSecSize outSize);
  62 static int      xmlSecOpenSSLKWDes3Decode                       (const xmlSecByte *key,
  63                                                                  xmlSecSize keySize,
  64                                                                  const xmlSecByte *in,
  65                                                                  xmlSecSize inSize,
  66                                                                  xmlSecByte *out,
  67                                                                  xmlSecSize outSize);
  68 static int      xmlSecOpenSSLKWDes3Encrypt                      (const xmlSecByte *key,
  69                                                                  xmlSecSize keySize,
  70                                                                  const xmlSecByte *iv,
  71                                                                  xmlSecSize ivSize,
  72                                                                  const xmlSecByte *in,
  73                                                                  xmlSecSize inSize,
  74                                                                  xmlSecByte *out,
  75                                                                  xmlSecSize outSize,
  76                                                                  int enc);
  77 static int      xmlSecOpenSSLKWDes3BufferReverse                (xmlSecByte *buf,
  78                                                                  xmlSecSize size);
  79
  80 static xmlSecTransformKlass xmlSecOpenSSLKWDes3Klass = {
  81     /* klass/object sizes */
  82     sizeof(xmlSecTransformKlass),               /* xmlSecSize klassSize */
  83     xmlSecOpenSSLKWDes3Size,                    /* xmlSecSize objSize */
  84
  85     xmlSecNameKWDes3,                           /* const xmlChar* name; */
  86     xmlSecHrefKWDes3,                           /* const xmlChar* href; */
  87     xmlSecTransformUsageEncryptionMethod,       /* xmlSecAlgorithmUsage usage; */
  88
  89     xmlSecOpenSSLKWDes3Initialize,              /* xmlSecTransformInitializeMethod initialize; */
  90     xmlSecOpenSSLKWDes3Finalize,                /* xmlSecTransformFinalizeMethod finalize; */
  91     NULL,                                       /* xmlSecTransformNodeReadMethod readNode; */
  92     NULL,                                       /* xmlSecTransformNodeWriteMethod writeNode; */
  93     xmlSecOpenSSLKWDes3SetKeyReq,               /* xmlSecTransformSetKeyMethod setKeyReq; */
  94     xmlSecOpenSSLKWDes3SetKey,                  /* xmlSecTransformSetKeyMethod setKey; */
  95     NULL,                                       /* xmlSecTransformValidateMethod validate; */
  96     xmlSecTransformDefaultGetDataType,          /* xmlSecTransformGetDataTypeMethod getDataType; */
  97     xmlSecTransformDefaultPushBin,              /* xmlSecTransformPushBinMethod pushBin; */
  98     xmlSecTransformDefaultPopBin,               /* xmlSecTransformPopBinMethod popBin; */
  99     NULL,                                       /* xmlSecTransformPushXmlMethod pushXml; */
 100     NULL,                                       /* xmlSecTransformPopXmlMethod popXml; */
 101     xmlSecOpenSSLKWDes3Execute,                 /* xmlSecTransformExecuteMethod execute; */
 102
 103     NULL,                                       /* void* reserved0; */
 104     NULL,                                       /* void* reserved1; */
 105 };
 106
 107 /**
 108  * xmlSecOpenSSLTransformKWDes3GetKlass:
 109  *
 110  * The Triple DES key wrapper transform klass.
 111  *
 112  * Returns: Triple DES key wrapper transform klass.
 113  */
 114 xmlSecTransformId
 115 xmlSecOpenSSLTransformKWDes3GetKlass(void) {
 116     return(&xmlSecOpenSSLKWDes3Klass);
 117 }
 118
 119 static int
 120 xmlSecOpenSSLKWDes3Initialize(xmlSecTransformPtr transform) {
 121     int ret;
 122
 123     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformKWDes3Id), -1);
 124     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLKWDes3Size), -1);
 125
 126     ret = xmlSecBufferInitialize(xmlSecOpenSSLKWDes3GetKey(transform), 0);
 127     if(ret < 0) {
 128         xmlSecError(XMLSEC_ERRORS_HERE,
 129                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 130                     "xmlSecBufferInitialize",
 131                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 132                     XMLSEC_ERRORS_NO_MESSAGE);
 133         return(-1);
 134     }
 135
 136     return(0);
 137 }
 138
 139 static void
 140 xmlSecOpenSSLKWDes3Finalize(xmlSecTransformPtr transform) {
 141     xmlSecAssert(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformKWDes3Id));
 142     xmlSecAssert(xmlSecTransformCheckSize(transform, xmlSecOpenSSLKWDes3Size));
 143
 144     if(xmlSecOpenSSLKWDes3GetKey(transform) != NULL) {
 145         xmlSecBufferFinalize(xmlSecOpenSSLKWDes3GetKey(transform));
 146     }
 147 }
 148
 149 static int
 150 xmlSecOpenSSLKWDes3SetKeyReq(xmlSecTransformPtr transform,  xmlSecKeyReqPtr keyReq) {
 151     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformKWDes3Id), -1);
 152     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 153     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLKWDes3Size), -1);
 154     xmlSecAssert2(keyReq != NULL, -1);
 155
 156     keyReq->keyId       = xmlSecOpenSSLKeyDataDesId;
 157     keyReq->keyType     = xmlSecKeyDataTypeSymmetric;
 158     if(transform->operation == xmlSecTransformOperationEncrypt) {
 159         keyReq->keyUsage= xmlSecKeyUsageEncrypt;
 160     } else {
 161         keyReq->keyUsage= xmlSecKeyUsageDecrypt;
 162     }
 163     keyReq->keyBitsSize = 8 * XMLSEC_OPENSSL_DES3_KEY_LENGTH;
 164     return(0);
 165 }
 166
 167 static int
 168 xmlSecOpenSSLKWDes3SetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
 169     xmlSecBufferPtr buffer;
 170     xmlSecSize keySize;
 171     int ret;
 172
 173     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformKWDes3Id), -1);
 174     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 175     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLKWDes3Size), -1);
 176     xmlSecAssert2(xmlSecOpenSSLKWDes3GetKey(transform) != NULL, -1);
 177     xmlSecAssert2(key != NULL, -1);
 178     xmlSecAssert2(xmlSecKeyDataCheckId(xmlSecKeyGetValue(key), xmlSecOpenSSLKeyDataDesId), -1);
 179
 180     buffer = xmlSecKeyDataBinaryValueGetBuffer(xmlSecKeyGetValue(key));
 181     xmlSecAssert2(buffer != NULL, -1);
 182
 183     keySize = xmlSecBufferGetSize(buffer);
 184     if(keySize < XMLSEC_OPENSSL_DES3_KEY_LENGTH) {
 185         xmlSecError(XMLSEC_ERRORS_HERE,
 186                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 187                     NULL,
 188                     XMLSEC_ERRORS_R_INVALID_KEY_DATA_SIZE,
 189                     "key length %d is not enough (%d expected)",
 190                     keySize, XMLSEC_OPENSSL_DES3_KEY_LENGTH);
 191         return(-1);
 192     }
 193
 194     ret = xmlSecBufferSetData(xmlSecOpenSSLKWDes3GetKey(transform),
 195                             xmlSecBufferGetData(buffer),
 196                             XMLSEC_OPENSSL_DES3_KEY_LENGTH);
 197     if(ret < 0) {
 198         xmlSecError(XMLSEC_ERRORS_HERE,
 199                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 200                     "xmlSecBufferSetData",
 201                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 202                     "size=%d", XMLSEC_OPENSSL_DES3_KEY_LENGTH);
 203         return(-1);
 204     }
 205
 206     return(0);
 207 }
 208
 209 static int
 210 xmlSecOpenSSLKWDes3Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
 211     xmlSecBufferPtr in, out, key;
 212     xmlSecSize inSize, outSize, keySize;
 213     int ret;
 214
 215     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformKWDes3Id), -1);
 216     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 217     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLKWDes3Size), -1);
 218     xmlSecAssert2(transformCtx != NULL, -1);
 219
 220     key = xmlSecOpenSSLKWDes3GetKey(transform);
 221     xmlSecAssert2(key != NULL, -1);
 222
 223     keySize = xmlSecBufferGetSize(key);
 224     xmlSecAssert2(keySize == XMLSEC_OPENSSL_DES3_KEY_LENGTH, -1);
 225
 226     in = &(transform->inBuf);
 227     out = &(transform->outBuf);
 228     inSize = xmlSecBufferGetSize(in);
 229     outSize = xmlSecBufferGetSize(out);
 230     xmlSecAssert2(outSize == 0, -1);
 231
 232     if(transform->status == xmlSecTransformStatusNone) {
 233         transform->status = xmlSecTransformStatusWorking;
 234     }
 235
 236     if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) {
 237         /* just do nothing */
 238     } else  if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
 239         if((inSize % XMLSEC_OPENSSL_DES3_BLOCK_LENGTH) != 0) {
 240             xmlSecError(XMLSEC_ERRORS_HERE,
 241                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 242                         NULL,
 243                         XMLSEC_ERRORS_R_INVALID_SIZE,
 244                         "%d bytes - not %d bytes aligned",
 245                         inSize, XMLSEC_OPENSSL_DES3_BLOCK_LENGTH);
 246             return(-1);
 247         }
 248
 249         if(transform->operation == xmlSecTransformOperationEncrypt) {
 250             /* the encoded key might be 16 bytes longer plus one block just in case */
 251             outSize = inSize + XMLSEC_OPENSSL_DES3_IV_LENGTH +
 252                                XMLSEC_OPENSSL_DES3_BLOCK_LENGTH +
 253                                XMLSEC_OPENSSL_DES3_BLOCK_LENGTH;
 254         } else {
 255             outSize = inSize + XMLSEC_OPENSSL_DES3_BLOCK_LENGTH;
 256         }
 257
 258         ret = xmlSecBufferSetMaxSize(out, outSize);
 259         if(ret < 0) {
 260             xmlSecError(XMLSEC_ERRORS_HERE,
 261                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 262                         "xmlSecBufferSetMaxSize",
 263                         XMLSEC_ERRORS_R_XMLSEC_FAILED,
 264                         "size=%d", outSize);
 265             return(-1);
 266         }
 267
 268         if(transform->operation == xmlSecTransformOperationEncrypt) {
 269             ret = xmlSecOpenSSLKWDes3Encode(xmlSecBufferGetData(key), keySize,
 270                                             xmlSecBufferGetData(in), inSize,
 271                                             xmlSecBufferGetData(out), outSize);
 272             if(ret < 0) {
 273                 xmlSecError(XMLSEC_ERRORS_HERE,
 274                             xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 275                             "xmlSecOpenSSLKWDes3Encode",
 276                             XMLSEC_ERRORS_R_XMLSEC_FAILED,
 277                             "key=%d,in=%d,out=%d",
 278                             keySize, inSize, outSize);
 279                 return(-1);
 280             }
 281             outSize = ret;
 282         } else {
 283             ret = xmlSecOpenSSLKWDes3Decode(xmlSecBufferGetData(key), keySize,
 284                                             xmlSecBufferGetData(in), inSize,
 285                                             xmlSecBufferGetData(out), outSize);
 286             if(ret < 0) {
 287                 xmlSecError(XMLSEC_ERRORS_HERE,
 288                             xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 289                             "xmlSecOpenSSLKWDes3Decode",
 290                             XMLSEC_ERRORS_R_XMLSEC_FAILED,
 291                             "key=%d,in=%d,out=%d",
 292                             keySize, inSize, outSize);
 293                 return(-1);
 294             }
 295             outSize = ret;
 296         }
 297
 298         ret = xmlSecBufferSetSize(out, outSize);
 299         if(ret < 0) {
 300             xmlSecError(XMLSEC_ERRORS_HERE,
 301                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 302                         "xmlSecBufferSetSize",
 303                         XMLSEC_ERRORS_R_XMLSEC_FAILED,
 304                         "size=%d", outSize);
 305             return(-1);
 306         }
 307
 308         ret = xmlSecBufferRemoveHead(in, inSize);
 309         if(ret < 0) {
 310             xmlSecError(XMLSEC_ERRORS_HERE,
 311                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 312                         "xmlSecBufferRemoveHead",
 313                         XMLSEC_ERRORS_R_XMLSEC_FAILED,
 314                         "size=%d", inSize);
 315             return(-1);
 316         }
 317
 318         transform->status = xmlSecTransformStatusFinished;
 319     } else if(transform->status == xmlSecTransformStatusFinished) {
 320         /* the only way we can get here is if there is no input */
 321         xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
 322     } else {
 323         xmlSecError(XMLSEC_ERRORS_HERE,
 324                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 325                     NULL,
 326                     XMLSEC_ERRORS_R_INVALID_STATUS,
 327                     "status=%d", transform->status);
 328         return(-1);
 329     }
 330     return(0);
 331 }
 332
 333 static xmlSecByte xmlSecOpenSSLKWDes3Iv[XMLSEC_OPENSSL_DES3_IV_LENGTH] = {
 334     0x4a, 0xdd, 0xa2, 0x2c, 0x79, 0xe8, 0x21, 0x05
 335 };
 336 /**
 337  * CMS Triple DES Key Wrap
 338  *
 339  * http://www.w3.org/TR/xmlenc-core/#sec-Alg-SymmetricKeyWrap
 340  *
 341  * The following algorithm wraps (encrypts) a key (the wrapped key, WK)
 342  * under a TRIPLEDES key-encryption-key (KEK) as specified in [CMS-Algorithms]:
 343  *
 344  * 1. Represent the key being wrapped as an octet sequence. If it is a
 345  *    TRIPLEDES key, this is 24 octets (192 bits) with odd parity bit as
 346  *    the bottom bit of each octet.
 347  * 2. Compute the CMS key checksum (section 5.6.1) call this CKS.
 348  * 3. Let WKCKS = WK || CKS, where || is concatenation.
 349  * 4. Generate 8 random octets [RANDOM] and call this IV.
 350  * 5. Encrypt WKCKS in CBC mode using KEK as the key and IV as the
 351  *    initialization vector. Call the results TEMP1.
 352  * 6. Left TEMP2 = IV || TEMP1.
 353  * 7. Reverse the order of the octets in TEMP2 and call the result TEMP3.
 354  * 8. Encrypt TEMP3 in CBC mode using the KEK and an initialization vector
 355  *    of 0x4adda22c79e82105. The resulting cipher text is the desired result.
 356  *    It is 40 octets long if a 168 bit key is being wrapped.
 357  *
 358  */
 359 static int
 360 xmlSecOpenSSLKWDes3Encode(const xmlSecByte *key, xmlSecSize keySize,
 361                         const xmlSecByte *in, xmlSecSize inSize,
 362                         xmlSecByte *out, xmlSecSize outSize) {
 363     xmlSecByte sha1[SHA_DIGEST_LENGTH];
 364     xmlSecByte iv[XMLSEC_OPENSSL_DES3_IV_LENGTH];
 365     xmlSecSize s;
 366     int ret;
 367
 368     xmlSecAssert2(key != NULL, -1);
 369     xmlSecAssert2(keySize == XMLSEC_OPENSSL_DES3_KEY_LENGTH, -1);
 370     xmlSecAssert2(in != NULL, -1);
 371     xmlSecAssert2(inSize > 0, -1);
 372     xmlSecAssert2(out != NULL, -1);
 373     xmlSecAssert2(outSize >= inSize + 16, -1);
 374
 375     /* step 2: calculate sha1 and CMS */
 376     if(SHA1(in, inSize, sha1) == NULL) {
 377         xmlSecError(XMLSEC_ERRORS_HERE,
 378                     NULL,
 379                     "SHA1",
 380                     XMLSEC_ERRORS_R_CRYPTO_FAILED,
 381                     XMLSEC_ERRORS_NO_MESSAGE);
 382         return(-1);
 383     }
 384
 385     /* step 3: construct WKCKS */
 386     memcpy(out, in, inSize);
 387     memcpy(out + inSize, sha1, XMLSEC_OPENSSL_DES3_BLOCK_LENGTH);
 388
 389     /* step 4: generate random iv */
 390     ret = RAND_bytes(iv, XMLSEC_OPENSSL_DES3_IV_LENGTH);
 391     if(ret != 1) {
 392         xmlSecError(XMLSEC_ERRORS_HERE,
 393                     NULL,
 394                     "RAND_bytes",
 395                     XMLSEC_ERRORS_R_CRYPTO_FAILED,
 396                     "ret=%d", ret);
 397         return(-1);
 398     }
 399
 400     /* step 5: first encryption, result is TEMP1 */
 401     ret = xmlSecOpenSSLKWDes3Encrypt(key, keySize,
 402                                     iv, XMLSEC_OPENSSL_DES3_IV_LENGTH,
 403                                     out, inSize + XMLSEC_OPENSSL_DES3_BLOCK_LENGTH,
 404                                     out, outSize, 1);
 405     if(ret < 0) {
 406         xmlSecError(XMLSEC_ERRORS_HERE,
 407                     NULL,
 408                     "xmlSecOpenSSLKWDes3Encrypt",
 409                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 410                     XMLSEC_ERRORS_NO_MESSAGE);
 411         return(-1);
 412     }
 413
 414     /* step 6: construct TEMP2=IV || TEMP1 */
 415     memmove(out + XMLSEC_OPENSSL_DES3_IV_LENGTH, out,
 416             inSize + XMLSEC_OPENSSL_DES3_IV_LENGTH);
 417     memcpy(out, iv, XMLSEC_OPENSSL_DES3_IV_LENGTH);
 418     s = ret + XMLSEC_OPENSSL_DES3_IV_LENGTH;
 419
 420     /* step 7: reverse octets order, result is TEMP3 */
 421     ret = xmlSecOpenSSLKWDes3BufferReverse(out, s);
 422     if(ret < 0) {
 423         xmlSecError(XMLSEC_ERRORS_HERE,
 424                     NULL,
 425                     "xmlSecOpenSSLKWDes3BufferReverse",
 426                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 427                     XMLSEC_ERRORS_NO_MESSAGE);
 428         return(-1);
 429     }
 430
 431     /* step 8: second encryption with static IV */
 432     ret = xmlSecOpenSSLKWDes3Encrypt(key, keySize,
 433                                     xmlSecOpenSSLKWDes3Iv, XMLSEC_OPENSSL_DES3_IV_LENGTH,
 434                                     out, s, out, outSize, 1);
 435     if(ret < 0) {
 436         xmlSecError(XMLSEC_ERRORS_HERE,
 437                     NULL,
 438                     "xmlSecOpenSSLKWDes3Encrypt",
 439                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 440                     XMLSEC_ERRORS_NO_MESSAGE);
 441         return(-1);
 442     }
 443     s = ret;
 444     return(s);
 445 }
 446
 447 /**
 448  * CMS Triple DES Key Wrap
 449  *
 450  * http://www.w3.org/TR/xmlenc-core/#sec-Alg-SymmetricKeyWrap
 451  *
 452  * The following algorithm unwraps (decrypts) a key as specified in
 453  * [CMS-Algorithms]:
 454  *
 455  * 1. Check if the length of the cipher text is reasonable given the key type.
 456  *    It must be 40 bytes for a 168 bit key and either 32, 40, or 48 bytes for
 457  *    a 128, 192, or 256 bit key. If the length is not supported or inconsistent
 458  *    with the algorithm for which the key is intended, return error.
 459  * 2. Decrypt the cipher text with TRIPLEDES in CBC mode using the KEK and
 460  *    an initialization vector (IV) of 0x4adda22c79e82105. Call the output TEMP3.
 461  * 3. Reverse the order of the octets in TEMP3 and call the result TEMP2.
 462  * 4. Decompose TEMP2 into IV, the first 8 octets, and TEMP1, the remaining
 463  *    octets.
 464  * 5. Decrypt TEMP1 using TRIPLEDES in CBC mode using the KEK and the IV found
 465  *    in the previous step. Call the result WKCKS.
 466  * 6. Decompose WKCKS. CKS is the last 8 octets and WK, the wrapped key, are
 467  *    those octets before the CKS.
 468  * 7. Calculate a CMS key checksum (section 5.6.1) over the WK and compare
 469  *    with the CKS extracted in the above step. If they are not equal, return
 470  *    error.
 471  * 8. WK is the wrapped key, now extracted for use in data decryption.
 472  */
 473 static int
 474 xmlSecOpenSSLKWDes3Decode(const xmlSecByte *key, xmlSecSize keySize,
 475                         const xmlSecByte *in, xmlSecSize inSize,
 476                         xmlSecByte *out, xmlSecSize outSize) {
 477     xmlSecByte sha1[SHA_DIGEST_LENGTH];
 478     xmlSecSize s;
 479     int ret;
 480
 481     xmlSecAssert2(key != NULL, -1);
 482     xmlSecAssert2(keySize == XMLSEC_OPENSSL_DES3_KEY_LENGTH, -1);
 483     xmlSecAssert2(in != NULL, -1);
 484     xmlSecAssert2(inSize > 0, -1);
 485     xmlSecAssert2(out != NULL, -1);
 486     xmlSecAssert2(outSize >= inSize, -1);
 487
 488     /* step 2: first decryption with static IV, result is TEMP3 */
 489     ret = xmlSecOpenSSLKWDes3Encrypt(key, keySize,
 490                                     xmlSecOpenSSLKWDes3Iv, XMLSEC_OPENSSL_DES3_IV_LENGTH,
 491                                     in, inSize, out, outSize, 0);
 492     if((ret < 0) || (ret < XMLSEC_OPENSSL_DES3_IV_LENGTH)) {
 493         xmlSecError(XMLSEC_ERRORS_HERE,
 494                     NULL,
 495                     "xmlSecOpenSSLKWDes3Encrypt",
 496                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 497                     XMLSEC_ERRORS_NO_MESSAGE);
 498         return(-1);
 499     }
 500     s = ret;
 501
 502     /* step 3: reverse octets order in TEMP3, result is TEMP2 */
 503     ret = xmlSecOpenSSLKWDes3BufferReverse(out, s);
 504     if(ret < 0) {
 505         xmlSecError(XMLSEC_ERRORS_HERE,
 506                     NULL,
 507                     "xmlSecOpenSSLKWDes3BufferReverse",
 508                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 509                     XMLSEC_ERRORS_NO_MESSAGE);
 510         return(-1);
 511     }
 512
 513     /* steps 4 and 5: get IV and decrypt second time, result is WKCKS */
 514     ret = xmlSecOpenSSLKWDes3Encrypt(key, keySize,
 515                                      out, XMLSEC_OPENSSL_DES3_IV_LENGTH,
 516                                      out + XMLSEC_OPENSSL_DES3_IV_LENGTH,
 517                                      s - XMLSEC_OPENSSL_DES3_IV_LENGTH,
 518                                      out, outSize, 0);
 519     if((ret < 0) || (ret < XMLSEC_OPENSSL_DES3_BLOCK_LENGTH)) {
 520         xmlSecError(XMLSEC_ERRORS_HERE,
 521                     NULL,
 522                     "xmlSecOpenSSLKWDes3Encrypt",
 523                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 524                     XMLSEC_ERRORS_NO_MESSAGE);
 525         return(-1);
 526     }
 527     s = ret - XMLSEC_OPENSSL_DES3_BLOCK_LENGTH;
 528
 529     /* steps 6 and 7: calculate SHA1 and validate it */
 530     if(SHA1(out, s, sha1) == NULL) {
 531         xmlSecError(XMLSEC_ERRORS_HERE,
 532                     NULL,
 533                     "SHA1",
 534                     XMLSEC_ERRORS_R_CRYPTO_FAILED,
 535                     XMLSEC_ERRORS_NO_MESSAGE);
 536         return(-1);
 537     }
 538
 539     if(memcmp(sha1, out + s, XMLSEC_OPENSSL_DES3_BLOCK_LENGTH) != 0) {
 540         xmlSecError(XMLSEC_ERRORS_HERE,
 541                     NULL,
 542                     NULL,
 543                     XMLSEC_ERRORS_R_INVALID_DATA,
 544                     "SHA1 does not match");
 545         return(-1);
 546     }
 547
 548     return(s);
 549 }
 550
 551 static int
 552 xmlSecOpenSSLKWDes3Encrypt(const xmlSecByte *key, xmlSecSize keySize,
 553                            const xmlSecByte *iv, xmlSecSize ivSize,
 554                            const xmlSecByte *in, xmlSecSize inSize,
 555                            xmlSecByte *out, xmlSecSize outSize, int enc) {
 556     EVP_CIPHER_CTX cipherCtx;
 557     int updateLen;
 558     int finalLen;
 559     int ret;
 560
 561     xmlSecAssert2(key != NULL, -1);
 562     xmlSecAssert2(keySize == (xmlSecSize)EVP_CIPHER_key_length(EVP_des_ede3_cbc()), -1);
 563     xmlSecAssert2(iv != NULL, -1);
 564     xmlSecAssert2(ivSize == (xmlSecSize)EVP_CIPHER_iv_length(EVP_des_ede3_cbc()), -1);
 565     xmlSecAssert2(in != NULL, -1);
 566     xmlSecAssert2(inSize > 0, -1);
 567     xmlSecAssert2(out != NULL, -1);
 568     xmlSecAssert2(outSize >= inSize, -1);
 569
 570     EVP_CIPHER_CTX_init(&cipherCtx);
 571     ret = EVP_CipherInit(&cipherCtx, EVP_des_ede3_cbc(), key, iv, enc);
 572     if(ret != 1) {
 573         xmlSecError(XMLSEC_ERRORS_HERE,
 574                     NULL,
 575                     "EVP_CipherInit",
 576                     XMLSEC_ERRORS_R_CRYPTO_FAILED,
 577                     XMLSEC_ERRORS_NO_MESSAGE);
 578         return(-1);
 579     }
 580
 581 #ifndef XMLSEC_OPENSSL_096
 582     EVP_CIPHER_CTX_set_padding(&cipherCtx, 0);
 583 #endif /* XMLSEC_OPENSSL_096 */
 584
 585     ret = EVP_CipherUpdate(&cipherCtx, out, &updateLen, in, inSize);
 586     if(ret != 1) {
 587         xmlSecError(XMLSEC_ERRORS_HERE,
 588                     NULL,
 589                     "EVP_CipherUpdate",
 590                     XMLSEC_ERRORS_R_CRYPTO_FAILED,
 591                     XMLSEC_ERRORS_NO_MESSAGE);
 592         return(-1);
 593     }
 594
 595     ret = EVP_CipherFinal(&cipherCtx, out + updateLen, &finalLen);
 596     if(ret != 1) {
 597         xmlSecError(XMLSEC_ERRORS_HERE,
 598                     NULL,
 599                     "EVP_CipherFinal",
 600                     XMLSEC_ERRORS_R_CRYPTO_FAILED,
 601                     XMLSEC_ERRORS_NO_MESSAGE);
 602         return(-1);
 603     }
 604     EVP_CIPHER_CTX_cleanup(&cipherCtx);
 605
 606     return(updateLen + finalLen);
 607 }
 608
 609 static int
 610 xmlSecOpenSSLKWDes3BufferReverse(xmlSecByte *buf, xmlSecSize size) {
 611     xmlSecSize s;
 612     xmlSecSize i;
 613     xmlSecByte c;
 614
 615     xmlSecAssert2(buf != NULL, -1);
 616
 617     s = size / 2;
 618     --size;
 619     for(i = 0; i < s; ++i) {
 620         c = buf[i];
 621         buf[i] = buf[size - i];
 622         buf[size - i] = c;
 623     }
 624     return(0);
 625 }
 626
 627 #endif /* XMLSEC_NO_DES */
 628