src/openssl/kt_rsa.c

   1 /**
   2  *
   3  * XMLSec library
   4  *
   5  * RSA Algorithms support
   6  *
   7  * This is free software; see Copyright file in the source
   8  * distribution for preciese wording.
   9  *
  10  * Copyright (C) 2002-2003 Aleksey Sanin <aleksey@aleksey.com>
  11  */
  12 #include "globals.h"
  13
  14 #ifndef XMLSEC_NO_RSA
  15
  16 #include <stdlib.h>
  17 #include <string.h>
  18
  19 #include <openssl/rsa.h>
  20 #include <openssl/evp.h>
  21 #include <openssl/sha.h>
  22 #include <openssl/objects.h>
  23
  24 #include <libxml/tree.h>
  25
  26 #include <xmlsec/xmlsec.h>
  27 #include <xmlsec/buffer.h>
  28 #include <xmlsec/xmltree.h>
  29 #include <xmlsec/keys.h>
  30 #include <xmlsec/transforms.h>
  31 #include <xmlsec/strings.h>
  32 #include <xmlsec/errors.h>
  33
  34 #include <xmlsec/openssl/crypto.h>
  35 #include <xmlsec/openssl/evp.h>
  36 #include <xmlsec/openssl/bn.h>
  37
  38 /**************************************************************************
  39  *
  40  * Internal OpenSSL RSA PKCS1 CTX
  41  *
  42  *************************************************************************/
  43 typedef struct _xmlSecOpenSSLRsaPkcs1Ctx        xmlSecOpenSSLRsaPkcs1Ctx,
  44                                                 *xmlSecOpenSSLRsaPkcs1CtxPtr;
  45 struct _xmlSecOpenSSLRsaPkcs1Ctx {
  46     EVP_PKEY*           pKey;
  47 };
  48
  49 /*********************************************************************
  50  *
  51  * RSA PKCS1 key transport transform
  52  *
  53  * xmlSecOpenSSLRsaPkcs1Ctx is located after xmlSecTransform
  54  *
  55  ********************************************************************/
  56 #define xmlSecOpenSSLRsaPkcs1Size       \
  57     (sizeof(xmlSecTransform) + sizeof(xmlSecOpenSSLRsaPkcs1Ctx))
  58 #define xmlSecOpenSSLRsaPkcs1GetCtx(transform) \
  59     ((xmlSecOpenSSLRsaPkcs1CtxPtr)(((xmlSecByte*)(transform)) + sizeof(xmlSecTransform)))
  60
  61 static int      xmlSecOpenSSLRsaPkcs1Initialize                 (xmlSecTransformPtr transform);
  62 static void     xmlSecOpenSSLRsaPkcs1Finalize                   (xmlSecTransformPtr transform);
  63 static int      xmlSecOpenSSLRsaPkcs1SetKeyReq                  (xmlSecTransformPtr transform,
  64                                                                  xmlSecKeyReqPtr keyReq);
  65 static int      xmlSecOpenSSLRsaPkcs1SetKey                     (xmlSecTransformPtr transform,
  66                                                                  xmlSecKeyPtr key);
  67 static int      xmlSecOpenSSLRsaPkcs1Execute                    (xmlSecTransformPtr transform,
  68                                                                  int last,
  69                                                                  xmlSecTransformCtxPtr transformCtx);
  70 static int      xmlSecOpenSSLRsaPkcs1Process                    (xmlSecTransformPtr transform,
  71                                                                  xmlSecTransformCtxPtr transformCtx);
  72
  73 static xmlSecTransformKlass xmlSecOpenSSLRsaPkcs1Klass = {
  74     /* klass/object sizes */
  75     sizeof(xmlSecTransformKlass),               /* xmlSecSize klassSize */
  76     xmlSecOpenSSLRsaPkcs1Size,                  /* xmlSecSize objSize */
  77
  78     xmlSecNameRsaPkcs1,                         /* const xmlChar* name; */
  79     xmlSecHrefRsaPkcs1,                         /* const xmlChar* href; */
  80     xmlSecTransformUsageEncryptionMethod,       /* xmlSecAlgorithmUsage usage; */
  81
  82     xmlSecOpenSSLRsaPkcs1Initialize,            /* xmlSecTransformInitializeMethod initialize; */
  83     xmlSecOpenSSLRsaPkcs1Finalize,              /* xmlSecTransformFinalizeMethod finalize; */
  84     NULL,                                       /* xmlSecTransformNodeReadMethod readNode; */
  85     NULL,                                       /* xmlSecTransformNodeWriteMethod writeNode; */
  86     xmlSecOpenSSLRsaPkcs1SetKeyReq,             /* xmlSecTransformSetKeyMethod setKeyReq; */
  87     xmlSecOpenSSLRsaPkcs1SetKey,                /* xmlSecTransformSetKeyMethod setKey; */
  88     NULL,                                       /* xmlSecTransformValidateMethod validate; */
  89     xmlSecTransformDefaultGetDataType,          /* xmlSecTransformGetDataTypeMethod getDataType; */
  90     xmlSecTransformDefaultPushBin,              /* xmlSecTransformPushBinMethod pushBin; */
  91     xmlSecTransformDefaultPopBin,               /* xmlSecTransformPopBinMethod popBin; */
  92     NULL,                                       /* xmlSecTransformPushXmlMethod pushXml; */
  93     NULL,                                       /* xmlSecTransformPopXmlMethod popXml; */
  94     xmlSecOpenSSLRsaPkcs1Execute,               /* xmlSecTransformExecuteMethod execute; */
  95
  96     NULL,                                       /* void* reserved0; */
  97     NULL,                                       /* void* reserved1; */
  98 };
  99
 100 /**
 101  * xmlSecOpenSSLTransformRsaPkcs1GetKlass:
 102  *
 103  * The RSA-PKCS1 key transport transform klass.
 104  *
 105  * Returns: RSA-PKCS1 key transport transform klass.
 106  */
 107 xmlSecTransformId
 108 xmlSecOpenSSLTransformRsaPkcs1GetKlass(void) {
 109     return(&xmlSecOpenSSLRsaPkcs1Klass);
 110 }
 111
 112 static int
 113 xmlSecOpenSSLRsaPkcs1Initialize(xmlSecTransformPtr transform) {
 114     xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
 115
 116     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
 117     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1);
 118
 119     ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
 120     xmlSecAssert2(ctx != NULL, -1);
 121
 122     memset(ctx, 0, sizeof(xmlSecOpenSSLRsaPkcs1Ctx));
 123     return(0);
 124 }
 125
 126 static void
 127 xmlSecOpenSSLRsaPkcs1Finalize(xmlSecTransformPtr transform) {
 128     xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
 129
 130     xmlSecAssert(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id));
 131     xmlSecAssert(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size));
 132
 133     ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
 134     xmlSecAssert(ctx != NULL);
 135
 136     if(ctx->pKey != NULL) {
 137         EVP_PKEY_free(ctx->pKey);
 138     }
 139     memset(ctx, 0, sizeof(xmlSecOpenSSLRsaPkcs1Ctx));
 140 }
 141
 142 static int
 143 xmlSecOpenSSLRsaPkcs1SetKeyReq(xmlSecTransformPtr transform,  xmlSecKeyReqPtr keyReq) {
 144     xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
 145
 146     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
 147     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 148     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1);
 149     xmlSecAssert2(keyReq != NULL, -1);
 150
 151     ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
 152     xmlSecAssert2(ctx != NULL, -1);
 153
 154     keyReq->keyId        = xmlSecOpenSSLKeyDataRsaId;
 155     if(transform->operation == xmlSecTransformOperationEncrypt) {
 156         keyReq->keyType  = xmlSecKeyDataTypePublic;
 157         keyReq->keyUsage = xmlSecKeyUsageEncrypt;
 158     } else {
 159         keyReq->keyType  = xmlSecKeyDataTypePrivate;
 160         keyReq->keyUsage = xmlSecKeyUsageDecrypt;
 161     }
 162     return(0);
 163 }
 164
 165 static int
 166 xmlSecOpenSSLRsaPkcs1SetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
 167     xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
 168     EVP_PKEY* pKey;
 169
 170     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
 171     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 172     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1);
 173     xmlSecAssert2(key != NULL, -1);
 174     xmlSecAssert2(xmlSecKeyDataCheckId(xmlSecKeyGetValue(key), xmlSecOpenSSLKeyDataRsaId), -1);
 175
 176     ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
 177     xmlSecAssert2(ctx != NULL, -1);
 178     xmlSecAssert2(ctx->pKey == NULL, -1);
 179
 180     pKey = xmlSecOpenSSLKeyDataRsaGetEvp(xmlSecKeyGetValue(key));
 181     if(pKey == NULL) {
 182         xmlSecError(XMLSEC_ERRORS_HERE,
 183                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 184                     "xmlSecOpenSSLKeyDataRsaGetEvp",
 185                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 186                     XMLSEC_ERRORS_NO_MESSAGE);
 187         return(-1);
 188     }
 189     xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1);
 190     xmlSecAssert2(pKey->pkey.rsa != NULL, -1);
 191
 192     ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey);
 193     if(ctx->pKey == NULL) {
 194         xmlSecError(XMLSEC_ERRORS_HERE,
 195                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 196                     "xmlSecOpenSSLEvpKeyDup",
 197                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 198                     XMLSEC_ERRORS_NO_MESSAGE);
 199         return(-1);
 200     }
 201
 202     return(0);
 203 }
 204
 205 static int
 206 xmlSecOpenSSLRsaPkcs1Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
 207     xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
 208     int ret;
 209
 210     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
 211     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 212     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1);
 213     xmlSecAssert2(transformCtx != NULL, -1);
 214
 215     ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
 216     xmlSecAssert2(ctx != NULL, -1);
 217     xmlSecAssert2(ctx->pKey != NULL, -1);
 218
 219     if(transform->status == xmlSecTransformStatusNone) {
 220         transform->status = xmlSecTransformStatusWorking;
 221     }
 222
 223     if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) {
 224         /* just do nothing */
 225     } else  if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
 226         ret = xmlSecOpenSSLRsaPkcs1Process(transform, transformCtx);
 227         if(ret < 0) {
 228             xmlSecError(XMLSEC_ERRORS_HERE,
 229                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 230                         "xmlSecOpenSSLRsaPkcs1Process",
 231                         XMLSEC_ERRORS_R_XMLSEC_FAILED,
 232                         XMLSEC_ERRORS_NO_MESSAGE);
 233             return(-1);
 234         }
 235         transform->status = xmlSecTransformStatusFinished;
 236     } else if(transform->status == xmlSecTransformStatusFinished) {
 237         /* the only way we can get here is if there is no input */
 238         xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
 239     } else {
 240         xmlSecError(XMLSEC_ERRORS_HERE,
 241                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 242                     NULL,
 243                     XMLSEC_ERRORS_R_INVALID_STATUS,
 244                     "status=%d", transform->status);
 245         return(-1);
 246     }
 247     return(0);
 248 }
 249
 250 static int
 251 xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
 252     xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
 253     xmlSecBufferPtr in, out;
 254     xmlSecSize inSize, outSize;
 255     xmlSecSize keySize;
 256     int ret;
 257
 258     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
 259     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 260     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1);
 261     xmlSecAssert2(transformCtx != NULL, -1);
 262
 263     ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
 264     xmlSecAssert2(ctx != NULL, -1);
 265     xmlSecAssert2(ctx->pKey != NULL, -1);
 266     xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1);
 267     xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1);
 268
 269     keySize = RSA_size(ctx->pKey->pkey.rsa);
 270     xmlSecAssert2(keySize > 0, -1);
 271
 272     in = &(transform->inBuf);
 273     out = &(transform->outBuf);
 274
 275     inSize = xmlSecBufferGetSize(in);
 276     outSize = xmlSecBufferGetSize(out);
 277     xmlSecAssert2(outSize == 0, -1);
 278
 279     /* the encoded size is equal to the keys size so we could not
 280      * process more than that */
 281     if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
 282         xmlSecError(XMLSEC_ERRORS_HERE,
 283                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 284                     NULL,
 285                     XMLSEC_ERRORS_R_INVALID_SIZE,
 286                     "%d when expected less than %d", inSize, keySize);
 287         return(-1);
 288     } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
 289         xmlSecError(XMLSEC_ERRORS_HERE,
 290                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 291                     NULL,
 292                     XMLSEC_ERRORS_R_INVALID_SIZE,
 293                     "%d when expected %d", inSize, keySize);
 294         return(-1);
 295     }
 296
 297     outSize = keySize;
 298     ret = xmlSecBufferSetMaxSize(out, outSize);
 299     if(ret < 0) {
 300         xmlSecError(XMLSEC_ERRORS_HERE,
 301                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 302                     "xmlSecBufferSetMaxSize",
 303                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 304                     "size=%d", outSize);
 305         return(-1);
 306     }
 307
 308     if(transform->operation == xmlSecTransformOperationEncrypt) {
 309         ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
 310                                 xmlSecBufferGetData(out),
 311                                 ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING);
 312         if(ret <= 0) {
 313             xmlSecError(XMLSEC_ERRORS_HERE,
 314                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 315                         "RSA_public_encrypt",
 316                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 317                         "size=%d", inSize);
 318             return(-1);
 319         }
 320         outSize = ret;
 321     } else {
 322         ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
 323                                 xmlSecBufferGetData(out),
 324                                 ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING);
 325         if(ret <= 0) {
 326             xmlSecError(XMLSEC_ERRORS_HERE,
 327                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 328                         "RSA_private_decrypt",
 329                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 330                         "size=%d", inSize);
 331             return(-1);
 332         }
 333         outSize = ret;
 334     }
 335
 336     ret = xmlSecBufferSetSize(out, outSize);
 337     if(ret < 0) {
 338         xmlSecError(XMLSEC_ERRORS_HERE,
 339                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 340                     "xmlSecBufferSetSize",
 341                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 342                     "size=%d", outSize);
 343         return(-1);
 344     }
 345
 346     ret = xmlSecBufferRemoveHead(in, inSize);
 347     if(ret < 0) {
 348         xmlSecError(XMLSEC_ERRORS_HERE,
 349                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 350                     "xmlSecBufferRemoveHead",
 351                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 352                     "size=%d", inSize);
 353         return(-1);
 354     }
 355
 356     return(0);
 357 }
 358
 359 /**************************************************************************
 360  *
 361  * Internal OpenSSL RSA OAPE CTX
 362  *
 363  *************************************************************************/
 364 typedef struct _xmlSecOpenSSLRsaOaepCtx         xmlSecOpenSSLRsaOaepCtx,
 365                                                 *xmlSecOpenSSLRsaOaepCtxPtr;
 366 struct _xmlSecOpenSSLRsaOaepCtx {
 367     EVP_PKEY*           pKey;
 368     xmlSecBuffer        oaepParams;
 369 };
 370
 371 /*********************************************************************
 372  *
 373  * RSA OAEP key transport transform
 374  *
 375  * xmlSecOpenSSLRsaOaepCtx is located after xmlSecTransform
 376  *
 377  ********************************************************************/
 378 #define xmlSecOpenSSLRsaOaepSize        \
 379     (sizeof(xmlSecTransform) + sizeof(xmlSecOpenSSLRsaOaepCtx))
 380 #define xmlSecOpenSSLRsaOaepGetCtx(transform) \
 381     ((xmlSecOpenSSLRsaOaepCtxPtr)(((xmlSecByte*)(transform)) + sizeof(xmlSecTransform)))
 382
 383 static int      xmlSecOpenSSLRsaOaepInitialize                  (xmlSecTransformPtr transform);
 384 static void     xmlSecOpenSSLRsaOaepFinalize                    (xmlSecTransformPtr transform);
 385 static int      xmlSecOpenSSLRsaOaepNodeRead                    (xmlSecTransformPtr transform,
 386                                                                  xmlNodePtr node,
 387                                                                  xmlSecTransformCtxPtr transformCtx);
 388 static int      xmlSecOpenSSLRsaOaepSetKeyReq                   (xmlSecTransformPtr transform,
 389                                                                  xmlSecKeyReqPtr keyReq);
 390 static int      xmlSecOpenSSLRsaOaepSetKey                      (xmlSecTransformPtr transform,
 391                                                                  xmlSecKeyPtr key);
 392 static int      xmlSecOpenSSLRsaOaepExecute                     (xmlSecTransformPtr transform,
 393                                                                  int last,
 394                                                                  xmlSecTransformCtxPtr transformCtx);
 395 static int      xmlSecOpenSSLRsaOaepProcess                     (xmlSecTransformPtr transform,
 396                                                                  xmlSecTransformCtxPtr transformCtx);
 397
 398 static xmlSecTransformKlass xmlSecOpenSSLRsaOaepKlass = {
 399     /* klass/object sizes */
 400     sizeof(xmlSecTransformKlass),               /* xmlSecSize klassSize */
 401     xmlSecOpenSSLRsaOaepSize,                   /* xmlSecSize objSize */
 402
 403     xmlSecNameRsaOaep,                          /* const xmlChar* name; */
 404     xmlSecHrefRsaOaep,                          /* const xmlChar* href; */
 405     xmlSecTransformUsageEncryptionMethod,       /* xmlSecAlgorithmUsage usage; */
 406
 407     xmlSecOpenSSLRsaOaepInitialize,             /* xmlSecTransformInitializeMethod initialize; */
 408     xmlSecOpenSSLRsaOaepFinalize,               /* xmlSecTransformFinalizeMethod finalize; */
 409     xmlSecOpenSSLRsaOaepNodeRead,               /* xmlSecTransformNodeReadMethod readNode; */
 410     NULL,                                       /* xmlSecTransformNodeWriteMethod writeNode; */
 411     xmlSecOpenSSLRsaOaepSetKeyReq,              /* xmlSecTransformSetKeyMethod setKeyReq; */
 412     xmlSecOpenSSLRsaOaepSetKey,                 /* xmlSecTransformSetKeyMethod setKey; */
 413     NULL,                                       /* xmlSecTransformValidateMethod validate; */
 414     xmlSecTransformDefaultGetDataType,          /* xmlSecTransformGetDataTypeMethod getDataType; */
 415     xmlSecTransformDefaultPushBin,              /* xmlSecTransformPushBinMethod pushBin; */
 416     xmlSecTransformDefaultPopBin,               /* xmlSecTransformPopBinMethod popBin; */
 417     NULL,                                       /* xmlSecTransformPushXmlMethod pushXml; */
 418     NULL,                                       /* xmlSecTransformPopXmlMethod popXml; */
 419     xmlSecOpenSSLRsaOaepExecute,                /* xmlSecTransformExecuteMethod execute; */
 420
 421     NULL,                                       /* void* reserved0; */
 422     NULL,                                       /* void* reserved1; */
 423 };
 424
 425 /**
 426  * xmlSecOpenSSLTransformRsaOaepGetKlass:
 427  *
 428  * The RSA-OAEP key transport transform klass.
 429  *
 430  * Returns: RSA-OAEP key transport transform klass.
 431  */
 432 xmlSecTransformId
 433 xmlSecOpenSSLTransformRsaOaepGetKlass(void) {
 434     return(&xmlSecOpenSSLRsaOaepKlass);
 435 }
 436
 437 static int
 438 xmlSecOpenSSLRsaOaepInitialize(xmlSecTransformPtr transform) {
 439     xmlSecOpenSSLRsaOaepCtxPtr ctx;
 440     int ret;
 441
 442     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
 443     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
 444
 445     ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
 446     xmlSecAssert2(ctx != NULL, -1);
 447
 448     memset(ctx, 0, sizeof(xmlSecOpenSSLRsaOaepCtx));
 449
 450     ret = xmlSecBufferInitialize(&(ctx->oaepParams), 0);
 451     if(ret < 0) {
 452         xmlSecError(XMLSEC_ERRORS_HERE,
 453                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 454                     "xmlSecBufferInitialize",
 455                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 456                     XMLSEC_ERRORS_NO_MESSAGE);
 457         return(-1);
 458     }
 459     return(0);
 460 }
 461
 462 static void
 463 xmlSecOpenSSLRsaOaepFinalize(xmlSecTransformPtr transform) {
 464     xmlSecOpenSSLRsaOaepCtxPtr ctx;
 465
 466     xmlSecAssert(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId));
 467     xmlSecAssert(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize));
 468
 469     ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
 470     xmlSecAssert(ctx != NULL);
 471
 472     if(ctx->pKey != NULL) {
 473         EVP_PKEY_free(ctx->pKey);
 474     }
 475     xmlSecBufferFinalize(&(ctx->oaepParams));
 476     memset(ctx, 0, sizeof(xmlSecOpenSSLRsaOaepCtx));
 477 }
 478
 479 static int
 480 xmlSecOpenSSLRsaOaepNodeRead(xmlSecTransformPtr transform, xmlNodePtr node, xmlSecTransformCtxPtr transformCtx) {
 481     xmlSecOpenSSLRsaOaepCtxPtr ctx;
 482     xmlNodePtr cur;
 483     int ret;
 484
 485     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
 486     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
 487     xmlSecAssert2(node != NULL, -1);
 488     xmlSecAssert2(transformCtx != NULL, -1);
 489
 490     ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
 491     xmlSecAssert2(ctx != NULL, -1);
 492     xmlSecAssert2(xmlSecBufferGetSize(&(ctx->oaepParams)) == 0, -1);
 493
 494     cur = xmlSecGetNextElementNode(node->children);
 495     if((cur != NULL) && xmlSecCheckNodeName(cur,  xmlSecNodeRsaOAEPparams, xmlSecEncNs)) {
 496         ret = xmlSecBufferBase64NodeContentRead(&(ctx->oaepParams), cur);
 497         if(ret < 0) {
 498             xmlSecError(XMLSEC_ERRORS_HERE,
 499                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 500                         "xmlSecBufferBase64NodeContentRead",
 501                         XMLSEC_ERRORS_R_XMLSEC_FAILED,
 502                         XMLSEC_ERRORS_NO_MESSAGE);
 503             return(-1);
 504         }
 505         cur = xmlSecGetNextElementNode(cur->next);
 506     }
 507
 508     if((cur != NULL) && xmlSecCheckNodeName(cur,  xmlSecNodeDigestMethod, xmlSecDSigNs)) {
 509         xmlChar* algorithm;
 510
 511         /* Algorithm attribute is required */
 512         algorithm = xmlGetProp(cur, xmlSecAttrAlgorithm);
 513         if(algorithm == NULL) {
 514             xmlSecError(XMLSEC_ERRORS_HERE,
 515                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 516                         xmlSecErrorsSafeString(xmlSecAttrAlgorithm),
 517                         XMLSEC_ERRORS_R_INVALID_NODE_ATTRIBUTE,
 518                         "node=%s",
 519                         xmlSecErrorsSafeString(xmlSecNodeGetName(cur)));
 520             return(-1);
 521         }
 522
 523         /* for now we support only sha1 */
 524         if(xmlStrcmp(algorithm, xmlSecHrefSha1) != 0) {
 525             xmlSecError(XMLSEC_ERRORS_HERE,
 526                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 527                         xmlSecErrorsSafeString(algorithm),
 528                         XMLSEC_ERRORS_R_INVALID_TRANSFORM,
 529                         "digest algorithm is not supported for rsa/oaep");
 530             xmlFree(algorithm);
 531             return(-1);
 532         }
 533         xmlFree(algorithm);
 534
 535         cur = xmlSecGetNextElementNode(cur->next);
 536     }
 537
 538     if(cur != NULL) {
 539         xmlSecError(XMLSEC_ERRORS_HERE,
 540                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 541                     xmlSecErrorsSafeString(xmlSecNodeGetName(cur)),
 542                     XMLSEC_ERRORS_R_UNEXPECTED_NODE,
 543                     XMLSEC_ERRORS_NO_MESSAGE);
 544         return(-1);
 545     }
 546
 547     return(0);
 548 }
 549
 550 static int
 551 xmlSecOpenSSLRsaOaepSetKeyReq(xmlSecTransformPtr transform,  xmlSecKeyReqPtr keyReq) {
 552     xmlSecOpenSSLRsaOaepCtxPtr ctx;
 553
 554     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
 555     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 556     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
 557     xmlSecAssert2(keyReq != NULL, -1);
 558
 559     ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
 560     xmlSecAssert2(ctx != NULL, -1);
 561
 562     keyReq->keyId        = xmlSecOpenSSLKeyDataRsaId;
 563     if(transform->operation == xmlSecTransformOperationEncrypt) {
 564         keyReq->keyType  = xmlSecKeyDataTypePublic;
 565         keyReq->keyUsage = xmlSecKeyUsageEncrypt;
 566     } else {
 567         keyReq->keyType  = xmlSecKeyDataTypePrivate;
 568         keyReq->keyUsage = xmlSecKeyUsageDecrypt;
 569     }
 570
 571     return(0);
 572 }
 573
 574 static int
 575 xmlSecOpenSSLRsaOaepSetKey(xmlSecTransformPtr transform, xmlSecKeyPtr key) {
 576     xmlSecOpenSSLRsaOaepCtxPtr ctx;
 577     EVP_PKEY* pKey;
 578
 579     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
 580     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 581     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
 582     xmlSecAssert2(key != NULL, -1);
 583     xmlSecAssert2(xmlSecKeyDataCheckId(xmlSecKeyGetValue(key), xmlSecOpenSSLKeyDataRsaId), -1);
 584
 585     ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
 586     xmlSecAssert2(ctx != NULL, -1);
 587     xmlSecAssert2(ctx->pKey == NULL, -1);
 588
 589     pKey = xmlSecOpenSSLKeyDataRsaGetEvp(xmlSecKeyGetValue(key));
 590     if(pKey == NULL) {
 591         xmlSecError(XMLSEC_ERRORS_HERE,
 592                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 593                     "xmlSecOpenSSLKeyDataRsaGetEvp",
 594                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 595                     XMLSEC_ERRORS_NO_MESSAGE);
 596         return(-1);
 597     }
 598     xmlSecAssert2(pKey->type == EVP_PKEY_RSA, -1);
 599     xmlSecAssert2(pKey->pkey.rsa != NULL, -1);
 600
 601     ctx->pKey = xmlSecOpenSSLEvpKeyDup(pKey);
 602     if(ctx->pKey == NULL) {
 603         xmlSecError(XMLSEC_ERRORS_HERE,
 604                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 605                     "xmlSecOpenSSLEvpKeyDup",
 606                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 607                     XMLSEC_ERRORS_NO_MESSAGE);
 608         return(-1);
 609     }
 610
 611     return(0);
 612 }
 613
 614 static int
 615 xmlSecOpenSSLRsaOaepExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
 616     xmlSecOpenSSLRsaOaepCtxPtr ctx;
 617     int ret;
 618
 619     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
 620     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 621     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
 622     xmlSecAssert2(transformCtx != NULL, -1);
 623
 624     ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
 625     xmlSecAssert2(ctx != NULL, -1);
 626     xmlSecAssert2(ctx->pKey != NULL, -1);
 627
 628     if(transform->status == xmlSecTransformStatusNone) {
 629         transform->status = xmlSecTransformStatusWorking;
 630     }
 631
 632     if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) {
 633         /* just do nothing */
 634     } else  if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
 635         ret = xmlSecOpenSSLRsaOaepProcess(transform, transformCtx);
 636         if(ret < 0) {
 637             xmlSecError(XMLSEC_ERRORS_HERE,
 638                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 639                         "xmlSecOpenSSLRsaOaepProcess",
 640                         XMLSEC_ERRORS_R_XMLSEC_FAILED,
 641                         XMLSEC_ERRORS_NO_MESSAGE);
 642             return(-1);
 643         }
 644         transform->status = xmlSecTransformStatusFinished;
 645     } else if(transform->status == xmlSecTransformStatusFinished) {
 646         /* the only way we can get here is if there is no input */
 647         xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
 648     } else {
 649         xmlSecError(XMLSEC_ERRORS_HERE,
 650                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 651                     NULL,
 652                     XMLSEC_ERRORS_R_INVALID_STATUS,
 653                     "status=%d", transform->status);
 654         return(-1);
 655     }
 656     return(0);
 657 }
 658
 659 static int
 660 xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
 661     xmlSecOpenSSLRsaOaepCtxPtr ctx;
 662     xmlSecSize paramsSize;
 663     xmlSecBufferPtr in, out;
 664     xmlSecSize inSize, outSize;
 665     xmlSecSize keySize;
 666     int ret;
 667
 668     xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
 669     xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
 670     xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
 671     xmlSecAssert2(transformCtx != NULL, -1);
 672
 673     ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
 674     xmlSecAssert2(ctx != NULL, -1);
 675     xmlSecAssert2(ctx->pKey != NULL, -1);
 676     xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1);
 677     xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1);
 678
 679     keySize = RSA_size(ctx->pKey->pkey.rsa);
 680     xmlSecAssert2(keySize > 0, -1);
 681
 682     in = &(transform->inBuf);
 683     out = &(transform->outBuf);
 684
 685     inSize = xmlSecBufferGetSize(in);
 686     outSize = xmlSecBufferGetSize(out);
 687     xmlSecAssert2(outSize == 0, -1);
 688
 689     /* the encoded size is equal to the keys size so we could not
 690      * process more than that */
 691     if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
 692         xmlSecError(XMLSEC_ERRORS_HERE,
 693                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 694                     NULL,
 695                     XMLSEC_ERRORS_R_INVALID_SIZE,
 696                     "%d when expected less than %d", inSize, keySize);
 697         return(-1);
 698     } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
 699         xmlSecError(XMLSEC_ERRORS_HERE,
 700                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 701                     NULL,
 702                     XMLSEC_ERRORS_R_INVALID_SIZE,
 703                     "%d when expected %d", inSize, keySize);
 704         return(-1);
 705     }
 706
 707     outSize = keySize;
 708     ret = xmlSecBufferSetMaxSize(out, outSize);
 709     if(ret < 0) {
 710         xmlSecError(XMLSEC_ERRORS_HERE,
 711                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 712                     "xmlSecBufferSetMaxSize",
 713                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 714                     "size=%d", outSize);
 715         return(-1);
 716     }
 717
 718     paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams));
 719     if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) {
 720         /* encode w/o OAEPParams --> simple */
 721         ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
 722                                 xmlSecBufferGetData(out),
 723                                 ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
 724         if(ret <= 0) {
 725             xmlSecError(XMLSEC_ERRORS_HERE,
 726                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 727                         "RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)",
 728                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 729                         XMLSEC_ERRORS_NO_MESSAGE);
 730             return(-1);
 731         }
 732         outSize = ret;
 733     } else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) {
 734         xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1);
 735
 736         /* add space for padding */
 737         ret = xmlSecBufferSetMaxSize(in, keySize);
 738         if(ret < 0) {
 739             xmlSecError(XMLSEC_ERRORS_HERE,
 740                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 741                         "xmlSecBufferSetMaxSize",
 742                         XMLSEC_ERRORS_R_XMLSEC_FAILED,
 743                         "size=%d", keySize);
 744             return(-1);
 745         }
 746
 747         /* add padding */
 748         ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize,
 749                                          xmlSecBufferGetData(in), inSize,
 750                                          xmlSecBufferGetData(&(ctx->oaepParams)),
 751                                          paramsSize);
 752         if(ret != 1) {
 753             xmlSecError(XMLSEC_ERRORS_HERE,
 754                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 755                         "RSA_padding_add_PKCS1_OAEP",
 756                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 757                         XMLSEC_ERRORS_NO_MESSAGE);
 758             return(-1);
 759         }
 760         inSize = keySize;
 761
 762         /* encode with OAEPParams */
 763         ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
 764                                 xmlSecBufferGetData(out),
 765                                 ctx->pKey->pkey.rsa, RSA_NO_PADDING);
 766         if(ret <= 0) {
 767             xmlSecError(XMLSEC_ERRORS_HERE,
 768                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 769                         "RSA_public_encrypt(RSA_NO_PADDING)",
 770                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 771                         XMLSEC_ERRORS_NO_MESSAGE);
 772             return(-1);
 773         }
 774         outSize = ret;
 775     } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) {
 776         ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
 777                                 xmlSecBufferGetData(out),
 778                                 ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
 779         if(ret <= 0) {
 780             xmlSecError(XMLSEC_ERRORS_HERE,
 781                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 782                         "RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)",
 783                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 784                         XMLSEC_ERRORS_NO_MESSAGE);
 785             return(-1);
 786         }
 787         outSize = ret;
 788     } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) {
 789         BIGNUM bn;
 790
 791         ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
 792                                 xmlSecBufferGetData(out),
 793                                 ctx->pKey->pkey.rsa, RSA_NO_PADDING);
 794         if(ret <= 0) {
 795             xmlSecError(XMLSEC_ERRORS_HERE,
 796                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 797                         "RSA_private_decrypt(RSA_NO_PADDING)",
 798                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 799                         XMLSEC_ERRORS_NO_MESSAGE);
 800             return(-1);
 801         }
 802         outSize = ret;
 803
 804         /*
 805          * the private decrypt w/o padding adds '0's at the begginning.
 806          * it's not clear for me can I simply skip all '0's from the
 807          * beggining so I have to do decode it back to BIGNUM and dump
 808          * buffer again
 809          */
 810         BN_init(&bn);
 811         if(BN_bin2bn(xmlSecBufferGetData(out), outSize, &bn) == NULL) {
 812             xmlSecError(XMLSEC_ERRORS_HERE,
 813                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 814                         "BN_bin2bn",
 815                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 816                         "size=%d", outSize);
 817             BN_clear_free(&bn);
 818             return(-1);
 819         }
 820
 821         ret = BN_bn2bin(&bn, xmlSecBufferGetData(out));
 822         if(ret <= 0) {
 823             xmlSecError(XMLSEC_ERRORS_HERE,
 824                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 825                         "BN_bn2bin",
 826                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 827                         XMLSEC_ERRORS_NO_MESSAGE);
 828             BN_clear_free(&bn);
 829             return(-1);
 830         }
 831         BN_clear_free(&bn);
 832         outSize = ret;
 833
 834         ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize,
 835                                            xmlSecBufferGetData(out), outSize,
 836                                            keySize,
 837                                            xmlSecBufferGetData(&(ctx->oaepParams)),
 838                                            paramsSize);
 839         if(ret < 0) {
 840             xmlSecError(XMLSEC_ERRORS_HERE,
 841                         xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 842                         "RSA_padding_check_PKCS1_OAEP",
 843                         XMLSEC_ERRORS_R_CRYPTO_FAILED,
 844                         XMLSEC_ERRORS_NO_MESSAGE);
 845             return(-1);
 846         }
 847         outSize = ret;
 848     } else {
 849         xmlSecAssert2("we could not be here" == NULL, -1);
 850         return(-1);
 851     }
 852
 853     ret = xmlSecBufferSetSize(out, outSize);
 854     if(ret < 0) {
 855         xmlSecError(XMLSEC_ERRORS_HERE,
 856                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 857                     "xmlSecBufferSetSize",
 858                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 859                     "size=%d", outSize);
 860         return(-1);
 861     }
 862
 863     ret = xmlSecBufferRemoveHead(in, inSize);
 864     if(ret < 0) {
 865         xmlSecError(XMLSEC_ERRORS_HERE,
 866                     xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
 867                     "xmlSecBufferRemoveHead",
 868                     XMLSEC_ERRORS_R_XMLSEC_FAILED,
 869                     "size=%d", inSize);
 870         return(-1);
 871     }
 872
 873     return(0);
 874 }
 875
 876 #endif /* XMLSEC_NO_RSA */
 877