arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/crypto.h>
  25 #include <linux/module.h>
  26 #include <linux/err.h>
  27 #include <crypto/algapi.h>
  28 #include <crypto/aes.h>
  29 #include <crypto/cryptd.h>
  30 #include <crypto/ctr.h>
  31 #include <asm/cpu_device_id.h>
  32 #include <asm/i387.h>
  33 #include <asm/aes.h>
  34 #include <asm/crypto/ablk_helper.h>
  35 #include <crypto/scatterwalk.h>
  36 #include <crypto/internal/aead.h>
  37 #include <linux/workqueue.h>
  38 #include <linux/spinlock.h>
  39
  40 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
  41 #define HAS_CTR
  42 #endif
  43
  44 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
  45 #define HAS_LRW
  46 #endif
  47
  48 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
  49 #define HAS_PCBC
  50 #endif
  51
  52 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
  53 #define HAS_XTS
  54 #endif
  55
  56 /* This data is stored at the end of the crypto_tfm struct.
  57  * It's a type of per "session" data storage location.
  58  * This needs to be 16 byte aligned.
  59  */
  60 struct aesni_rfc4106_gcm_ctx {
  61         u8 hash_subkey[16];
  62         struct crypto_aes_ctx aes_key_expanded;
  63         u8 nonce[4];
  64         struct cryptd_aead *cryptd_tfm;
  65 };
  66
  67 struct aesni_gcm_set_hash_subkey_result {
  68         int err;
  69         struct completion completion;
  70 };
  71
  72 struct aesni_hash_subkey_req_data {
  73         u8 iv[16];
  74         struct aesni_gcm_set_hash_subkey_result result;
  75         struct scatterlist sg;
  76 };
  77
  78 #define AESNI_ALIGN     (16)
  79 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
  80 #define RFC4106_HASH_SUBKEY_SIZE 16
  81
  82 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  83                              unsigned int key_len);
  84 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  85                           const u8 *in);
  86 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  87                           const u8 *in);
  88 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  89                               const u8 *in, unsigned int len);
  90 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  91                               const u8 *in, unsigned int len);
  92 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  93                               const u8 *in, unsigned int len, u8 *iv);
  94 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  95                               const u8 *in, unsigned int len, u8 *iv);
  96
  97 int crypto_fpu_init(void);
  98 void crypto_fpu_exit(void);
  99
 100 #ifdef CONFIG_X86_64
 101 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 102                               const u8 *in, unsigned int len, u8 *iv);
 103
 104 /* asmlinkage void aesni_gcm_enc()
 105  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 106  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 107  * const u8 *in, Plaintext input
 108  * unsigned long plaintext_len, Length of data in bytes for encryption.
 109  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 110  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 111  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 112  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 113  * const u8 *aad, Additional Authentication Data (AAD)
 114  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
 115  *          is going to be 8 or 12 bytes
 116  * u8 *auth_tag, Authenticated Tag output.
 117  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 118  *          Valid values are 16 (most likely), 12 or 8.
 119  */
 120 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 121                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 122                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 123                         u8 *auth_tag, unsigned long auth_tag_len);
 124
 125 /* asmlinkage void aesni_gcm_dec()
 126  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 127  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 128  * const u8 *in, Ciphertext input
 129  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 130  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 131  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 132  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 133  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 134  * const u8 *aad, Additional Authentication Data (AAD)
 135  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 136  * to be 8 or 12 bytes
 137  * u8 *auth_tag, Authenticated Tag output.
 138  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 139  * Valid values are 16 (most likely), 12 or 8.
 140  */
 141 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 142                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 143                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 144                         u8 *auth_tag, unsigned long auth_tag_len);
 145
 146 static inline struct
 147 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 148 {
 149         return
 150                 (struct aesni_rfc4106_gcm_ctx *)
 151                 PTR_ALIGN((u8 *)
 152                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
 153 }
 154 #endif
 155
 156 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 157 {
 158         unsigned long addr = (unsigned long)raw_ctx;
 159         unsigned long align = AESNI_ALIGN;
 160
 161         if (align <= crypto_tfm_ctx_alignment())
 162                 align = 1;
 163         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 164 }
 165
 166 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 167                               const u8 *in_key, unsigned int key_len)
 168 {
 169         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 170         u32 *flags = &tfm->crt_flags;
 171         int err;
 172
 173         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 174             key_len != AES_KEYSIZE_256) {
 175                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 176                 return -EINVAL;
 177         }
 178
 179         if (!irq_fpu_usable())
 180                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 181         else {
 182                 kernel_fpu_begin();
 183                 err = aesni_set_key(ctx, in_key, key_len);
 184                 kernel_fpu_end();
 185         }
 186
 187         return err;
 188 }
 189
 190 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 191                        unsigned int key_len)
 192 {
 193         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 194 }
 195
 196 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 197 {
 198         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 199
 200         if (!irq_fpu_usable())
 201                 crypto_aes_encrypt_x86(ctx, dst, src);
 202         else {
 203                 kernel_fpu_begin();
 204                 aesni_enc(ctx, dst, src);
 205                 kernel_fpu_end();
 206         }
 207 }
 208
 209 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 210 {
 211         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 212
 213         if (!irq_fpu_usable())
 214                 crypto_aes_decrypt_x86(ctx, dst, src);
 215         else {
 216                 kernel_fpu_begin();
 217                 aesni_dec(ctx, dst, src);
 218                 kernel_fpu_end();
 219         }
 220 }
 221
 222 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 223 {
 224         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 225
 226         aesni_enc(ctx, dst, src);
 227 }
 228
 229 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 230 {
 231         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 232
 233         aesni_dec(ctx, dst, src);
 234 }
 235
 236 static int ecb_encrypt(struct blkcipher_desc *desc,
 237                        struct scatterlist *dst, struct scatterlist *src,
 238                        unsigned int nbytes)
 239 {
 240         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 241         struct blkcipher_walk walk;
 242         int err;
 243
 244         blkcipher_walk_init(&walk, dst, src, nbytes);
 245         err = blkcipher_walk_virt(desc, &walk);
 246         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 247
 248         kernel_fpu_begin();
 249         while ((nbytes = walk.nbytes)) {
 250                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 251                               nbytes & AES_BLOCK_MASK);
 252                 nbytes &= AES_BLOCK_SIZE - 1;
 253                 err = blkcipher_walk_done(desc, &walk, nbytes);
 254         }
 255         kernel_fpu_end();
 256
 257         return err;
 258 }
 259
 260 static int ecb_decrypt(struct blkcipher_desc *desc,
 261                        struct scatterlist *dst, struct scatterlist *src,
 262                        unsigned int nbytes)
 263 {
 264         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 265         struct blkcipher_walk walk;
 266         int err;
 267
 268         blkcipher_walk_init(&walk, dst, src, nbytes);
 269         err = blkcipher_walk_virt(desc, &walk);
 270         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 271
 272         kernel_fpu_begin();
 273         while ((nbytes = walk.nbytes)) {
 274                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 275                               nbytes & AES_BLOCK_MASK);
 276                 nbytes &= AES_BLOCK_SIZE - 1;
 277                 err = blkcipher_walk_done(desc, &walk, nbytes);
 278         }
 279         kernel_fpu_end();
 280
 281         return err;
 282 }
 283
 284 static int cbc_encrypt(struct blkcipher_desc *desc,
 285                        struct scatterlist *dst, struct scatterlist *src,
 286                        unsigned int nbytes)
 287 {
 288         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 289         struct blkcipher_walk walk;
 290         int err;
 291
 292         blkcipher_walk_init(&walk, dst, src, nbytes);
 293         err = blkcipher_walk_virt(desc, &walk);
 294         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 295
 296         kernel_fpu_begin();
 297         while ((nbytes = walk.nbytes)) {
 298                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 299                               nbytes & AES_BLOCK_MASK, walk.iv);
 300                 nbytes &= AES_BLOCK_SIZE - 1;
 301                 err = blkcipher_walk_done(desc, &walk, nbytes);
 302         }
 303         kernel_fpu_end();
 304
 305         return err;
 306 }
 307
 308 static int cbc_decrypt(struct blkcipher_desc *desc,
 309                        struct scatterlist *dst, struct scatterlist *src,
 310                        unsigned int nbytes)
 311 {
 312         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 313         struct blkcipher_walk walk;
 314         int err;
 315
 316         blkcipher_walk_init(&walk, dst, src, nbytes);
 317         err = blkcipher_walk_virt(desc, &walk);
 318         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 319
 320         kernel_fpu_begin();
 321         while ((nbytes = walk.nbytes)) {
 322                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 323                               nbytes & AES_BLOCK_MASK, walk.iv);
 324                 nbytes &= AES_BLOCK_SIZE - 1;
 325                 err = blkcipher_walk_done(desc, &walk, nbytes);
 326         }
 327         kernel_fpu_end();
 328
 329         return err;
 330 }
 331
 332 #ifdef CONFIG_X86_64
 333 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 334                             struct blkcipher_walk *walk)
 335 {
 336         u8 *ctrblk = walk->iv;
 337         u8 keystream[AES_BLOCK_SIZE];
 338         u8 *src = walk->src.virt.addr;
 339         u8 *dst = walk->dst.virt.addr;
 340         unsigned int nbytes = walk->nbytes;
 341
 342         aesni_enc(ctx, keystream, ctrblk);
 343         crypto_xor(keystream, src, nbytes);
 344         memcpy(dst, keystream, nbytes);
 345         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 346 }
 347
 348 static int ctr_crypt(struct blkcipher_desc *desc,
 349                      struct scatterlist *dst, struct scatterlist *src,
 350                      unsigned int nbytes)
 351 {
 352         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 353         struct blkcipher_walk walk;
 354         int err;
 355
 356         blkcipher_walk_init(&walk, dst, src, nbytes);
 357         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 358         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 359
 360         kernel_fpu_begin();
 361         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 362                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 363                               nbytes & AES_BLOCK_MASK, walk.iv);
 364                 nbytes &= AES_BLOCK_SIZE - 1;
 365                 err = blkcipher_walk_done(desc, &walk, nbytes);
 366         }
 367         if (walk.nbytes) {
 368                 ctr_crypt_final(ctx, &walk);
 369                 err = blkcipher_walk_done(desc, &walk, 0);
 370         }
 371         kernel_fpu_end();
 372
 373         return err;
 374 }
 375 #endif
 376
 377 static int ablk_ecb_init(struct crypto_tfm *tfm)
 378 {
 379         return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
 380 }
 381
 382 static int ablk_cbc_init(struct crypto_tfm *tfm)
 383 {
 384         return ablk_init_common(tfm, "__driver-cbc-aes-aesni");
 385 }
 386
 387 #ifdef CONFIG_X86_64
 388 static int ablk_ctr_init(struct crypto_tfm *tfm)
 389 {
 390         return ablk_init_common(tfm, "__driver-ctr-aes-aesni");
 391 }
 392
 393 #ifdef HAS_CTR
 394 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
 395 {
 396         return ablk_init_common(tfm, "rfc3686(__driver-ctr-aes-aesni)");
 397 }
 398 #endif
 399 #endif
 400
 401 #ifdef HAS_LRW
 402 static int ablk_lrw_init(struct crypto_tfm *tfm)
 403 {
 404         return ablk_init_common(tfm, "fpu(lrw(__driver-aes-aesni))");
 405 }
 406 #endif
 407
 408 #ifdef HAS_PCBC
 409 static int ablk_pcbc_init(struct crypto_tfm *tfm)
 410 {
 411         return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))");
 412 }
 413 #endif
 414
 415 #ifdef HAS_XTS
 416 static int ablk_xts_init(struct crypto_tfm *tfm)
 417 {
 418         return ablk_init_common(tfm, "fpu(xts(__driver-aes-aesni))");
 419 }
 420 #endif
 421
 422 #ifdef CONFIG_X86_64
 423 static int rfc4106_init(struct crypto_tfm *tfm)
 424 {
 425         struct cryptd_aead *cryptd_tfm;
 426         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
 427                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 428         struct crypto_aead *cryptd_child;
 429         struct aesni_rfc4106_gcm_ctx *child_ctx;
 430         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
 431         if (IS_ERR(cryptd_tfm))
 432                 return PTR_ERR(cryptd_tfm);
 433
 434         cryptd_child = cryptd_aead_child(cryptd_tfm);
 435         child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
 436         memcpy(child_ctx, ctx, sizeof(*ctx));
 437         ctx->cryptd_tfm = cryptd_tfm;
 438         tfm->crt_aead.reqsize = sizeof(struct aead_request)
 439                 + crypto_aead_reqsize(&cryptd_tfm->base);
 440         return 0;
 441 }
 442
 443 static void rfc4106_exit(struct crypto_tfm *tfm)
 444 {
 445         struct aesni_rfc4106_gcm_ctx *ctx =
 446                 (struct aesni_rfc4106_gcm_ctx *)
 447                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 448         if (!IS_ERR(ctx->cryptd_tfm))
 449                 cryptd_free_aead(ctx->cryptd_tfm);
 450         return;
 451 }
 452
 453 static void
 454 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
 455 {
 456         struct aesni_gcm_set_hash_subkey_result *result = req->data;
 457
 458         if (err == -EINPROGRESS)
 459                 return;
 460         result->err = err;
 461         complete(&result->completion);
 462 }
 463
 464 static int
 465 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 466 {
 467         struct crypto_ablkcipher *ctr_tfm;
 468         struct ablkcipher_request *req;
 469         int ret = -EINVAL;
 470         struct aesni_hash_subkey_req_data *req_data;
 471
 472         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
 473         if (IS_ERR(ctr_tfm))
 474                 return PTR_ERR(ctr_tfm);
 475
 476         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
 477
 478         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
 479         if (ret)
 480                 goto out_free_ablkcipher;
 481
 482         ret = -ENOMEM;
 483         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
 484         if (!req)
 485                 goto out_free_ablkcipher;
 486
 487         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
 488         if (!req_data)
 489                 goto out_free_request;
 490
 491         memset(req_data->iv, 0, sizeof(req_data->iv));
 492
 493         /* Clear the data in the hash sub key container to zero.*/
 494         /* We want to cipher all zeros to create the hash sub key. */
 495         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 496
 497         init_completion(&req_data->result.completion);
 498         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
 499         ablkcipher_request_set_tfm(req, ctr_tfm);
 500         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
 501                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
 502                                         rfc4106_set_hash_subkey_done,
 503                                         &req_data->result);
 504
 505         ablkcipher_request_set_crypt(req, &req_data->sg,
 506                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
 507
 508         ret = crypto_ablkcipher_encrypt(req);
 509         if (ret == -EINPROGRESS || ret == -EBUSY) {
 510                 ret = wait_for_completion_interruptible
 511                         (&req_data->result.completion);
 512                 if (!ret)
 513                         ret = req_data->result.err;
 514         }
 515         kfree(req_data);
 516 out_free_request:
 517         ablkcipher_request_free(req);
 518 out_free_ablkcipher:
 519         crypto_free_ablkcipher(ctr_tfm);
 520         return ret;
 521 }
 522
 523 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
 524                                                    unsigned int key_len)
 525 {
 526         int ret = 0;
 527         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
 528         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 529         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 530         struct aesni_rfc4106_gcm_ctx *child_ctx =
 531                                  aesni_rfc4106_gcm_ctx_get(cryptd_child);
 532         u8 *new_key_mem = NULL;
 533
 534         if (key_len < 4) {
 535                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 536                 return -EINVAL;
 537         }
 538         /*Account for 4 byte nonce at the end.*/
 539         key_len -= 4;
 540         if (key_len != AES_KEYSIZE_128) {
 541                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 542                 return -EINVAL;
 543         }
 544
 545         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 546         /*This must be on a 16 byte boundary!*/
 547         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
 548                 return -EINVAL;
 549
 550         if ((unsigned long)key % AESNI_ALIGN) {
 551                 /*key is not aligned: use an auxuliar aligned pointer*/
 552                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
 553                 if (!new_key_mem)
 554                         return -ENOMEM;
 555
 556                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
 557                 memcpy(new_key_mem, key, key_len);
 558                 key = new_key_mem;
 559         }
 560
 561         if (!irq_fpu_usable())
 562                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
 563                 key, key_len);
 564         else {
 565                 kernel_fpu_begin();
 566                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
 567                 kernel_fpu_end();
 568         }
 569         /*This must be on a 16 byte boundary!*/
 570         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
 571                 ret = -EINVAL;
 572                 goto exit;
 573         }
 574         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 575         memcpy(child_ctx, ctx, sizeof(*ctx));
 576 exit:
 577         kfree(new_key_mem);
 578         return ret;
 579 }
 580
 581 /* This is the Integrity Check Value (aka the authentication tag length and can
 582  * be 8, 12 or 16 bytes long. */
 583 static int rfc4106_set_authsize(struct crypto_aead *parent,
 584                                 unsigned int authsize)
 585 {
 586         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 587         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 588
 589         switch (authsize) {
 590         case 8:
 591         case 12:
 592         case 16:
 593                 break;
 594         default:
 595                 return -EINVAL;
 596         }
 597         crypto_aead_crt(parent)->authsize = authsize;
 598         crypto_aead_crt(cryptd_child)->authsize = authsize;
 599         return 0;
 600 }
 601
 602 static int rfc4106_encrypt(struct aead_request *req)
 603 {
 604         int ret;
 605         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 606         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 607
 608         if (!irq_fpu_usable()) {
 609                 struct aead_request *cryptd_req =
 610                         (struct aead_request *) aead_request_ctx(req);
 611                 memcpy(cryptd_req, req, sizeof(*req));
 612                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 613                 return crypto_aead_encrypt(cryptd_req);
 614         } else {
 615                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 616                 kernel_fpu_begin();
 617                 ret = cryptd_child->base.crt_aead.encrypt(req);
 618                 kernel_fpu_end();
 619                 return ret;
 620         }
 621 }
 622
 623 static int rfc4106_decrypt(struct aead_request *req)
 624 {
 625         int ret;
 626         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 627         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 628
 629         if (!irq_fpu_usable()) {
 630                 struct aead_request *cryptd_req =
 631                         (struct aead_request *) aead_request_ctx(req);
 632                 memcpy(cryptd_req, req, sizeof(*req));
 633                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 634                 return crypto_aead_decrypt(cryptd_req);
 635         } else {
 636                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 637                 kernel_fpu_begin();
 638                 ret = cryptd_child->base.crt_aead.decrypt(req);
 639                 kernel_fpu_end();
 640                 return ret;
 641         }
 642 }
 643
 644 static int __driver_rfc4106_encrypt(struct aead_request *req)
 645 {
 646         u8 one_entry_in_sg = 0;
 647         u8 *src, *dst, *assoc;
 648         __be32 counter = cpu_to_be32(1);
 649         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 650         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 651         void *aes_ctx = &(ctx->aes_key_expanded);
 652         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 653         u8 iv_tab[16+AESNI_ALIGN];
 654         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
 655         struct scatter_walk src_sg_walk;
 656         struct scatter_walk assoc_sg_walk;
 657         struct scatter_walk dst_sg_walk;
 658         unsigned int i;
 659
 660         /* Assuming we are supporting rfc4106 64-bit extended */
 661         /* sequence numbers We need to have the AAD length equal */
 662         /* to 8 or 12 bytes */
 663         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
 664                 return -EINVAL;
 665         /* IV below built */
 666         for (i = 0; i < 4; i++)
 667                 *(iv+i) = ctx->nonce[i];
 668         for (i = 0; i < 8; i++)
 669                 *(iv+4+i) = req->iv[i];
 670         *((__be32 *)(iv+12)) = counter;
 671
 672         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
 673                 one_entry_in_sg = 1;
 674                 scatterwalk_start(&src_sg_walk, req->src);
 675                 scatterwalk_start(&assoc_sg_walk, req->assoc);
 676                 src = scatterwalk_map(&src_sg_walk);
 677                 assoc = scatterwalk_map(&assoc_sg_walk);
 678                 dst = src;
 679                 if (unlikely(req->src != req->dst)) {
 680                         scatterwalk_start(&dst_sg_walk, req->dst);
 681                         dst = scatterwalk_map(&dst_sg_walk);
 682                 }
 683
 684         } else {
 685                 /* Allocate memory for src, dst, assoc */
 686                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
 687                         GFP_ATOMIC);
 688                 if (unlikely(!src))
 689                         return -ENOMEM;
 690                 assoc = (src + req->cryptlen + auth_tag_len);
 691                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
 692                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
 693                                         req->assoclen, 0);
 694                 dst = src;
 695         }
 696
 697         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
 698                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
 699                 + ((unsigned long)req->cryptlen), auth_tag_len);
 700
 701         /* The authTag (aka the Integrity Check Value) needs to be written
 702          * back to the packet. */
 703         if (one_entry_in_sg) {
 704                 if (unlikely(req->src != req->dst)) {
 705                         scatterwalk_unmap(dst);
 706                         scatterwalk_done(&dst_sg_walk, 0, 0);
 707                 }
 708                 scatterwalk_unmap(src);
 709                 scatterwalk_unmap(assoc);
 710                 scatterwalk_done(&src_sg_walk, 0, 0);
 711                 scatterwalk_done(&assoc_sg_walk, 0, 0);
 712         } else {
 713                 scatterwalk_map_and_copy(dst, req->dst, 0,
 714                         req->cryptlen + auth_tag_len, 1);
 715                 kfree(src);
 716         }
 717         return 0;
 718 }
 719
 720 static int __driver_rfc4106_decrypt(struct aead_request *req)
 721 {
 722         u8 one_entry_in_sg = 0;
 723         u8 *src, *dst, *assoc;
 724         unsigned long tempCipherLen = 0;
 725         __be32 counter = cpu_to_be32(1);
 726         int retval = 0;
 727         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 728         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 729         void *aes_ctx = &(ctx->aes_key_expanded);
 730         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 731         u8 iv_and_authTag[32+AESNI_ALIGN];
 732         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
 733         u8 *authTag = iv + 16;
 734         struct scatter_walk src_sg_walk;
 735         struct scatter_walk assoc_sg_walk;
 736         struct scatter_walk dst_sg_walk;
 737         unsigned int i;
 738
 739         if (unlikely((req->cryptlen < auth_tag_len) ||
 740                 (req->assoclen != 8 && req->assoclen != 12)))
 741                 return -EINVAL;
 742         /* Assuming we are supporting rfc4106 64-bit extended */
 743         /* sequence numbers We need to have the AAD length */
 744         /* equal to 8 or 12 bytes */
 745
 746         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
 747         /* IV below built */
 748         for (i = 0; i < 4; i++)
 749                 *(iv+i) = ctx->nonce[i];
 750         for (i = 0; i < 8; i++)
 751                 *(iv+4+i) = req->iv[i];
 752         *((__be32 *)(iv+12)) = counter;
 753
 754         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
 755                 one_entry_in_sg = 1;
 756                 scatterwalk_start(&src_sg_walk, req->src);
 757                 scatterwalk_start(&assoc_sg_walk, req->assoc);
 758                 src = scatterwalk_map(&src_sg_walk);
 759                 assoc = scatterwalk_map(&assoc_sg_walk);
 760                 dst = src;
 761                 if (unlikely(req->src != req->dst)) {
 762                         scatterwalk_start(&dst_sg_walk, req->dst);
 763                         dst = scatterwalk_map(&dst_sg_walk);
 764                 }
 765
 766         } else {
 767                 /* Allocate memory for src, dst, assoc */
 768                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
 769                 if (!src)
 770                         return -ENOMEM;
 771                 assoc = (src + req->cryptlen + auth_tag_len);
 772                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
 773                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
 774                         req->assoclen, 0);
 775                 dst = src;
 776         }
 777
 778         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
 779                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
 780                 authTag, auth_tag_len);
 781
 782         /* Compare generated tag with passed in tag. */
 783         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
 784                 -EBADMSG : 0;
 785
 786         if (one_entry_in_sg) {
 787                 if (unlikely(req->src != req->dst)) {
 788                         scatterwalk_unmap(dst);
 789                         scatterwalk_done(&dst_sg_walk, 0, 0);
 790                 }
 791                 scatterwalk_unmap(src);
 792                 scatterwalk_unmap(assoc);
 793                 scatterwalk_done(&src_sg_walk, 0, 0);
 794                 scatterwalk_done(&assoc_sg_walk, 0, 0);
 795         } else {
 796                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
 797                 kfree(src);
 798         }
 799         return retval;
 800 }
 801 #endif
 802
 803 static struct crypto_alg aesni_algs[] = { {
 804         .cra_name               = "aes",
 805         .cra_driver_name        = "aes-aesni",
 806         .cra_priority           = 300,
 807         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 808         .cra_blocksize          = AES_BLOCK_SIZE,
 809         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
 810                                   AESNI_ALIGN - 1,
 811         .cra_alignmask          = 0,
 812         .cra_module             = THIS_MODULE,
 813         .cra_u  = {
 814                 .cipher = {
 815                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 816                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 817                         .cia_setkey             = aes_set_key,
 818                         .cia_encrypt            = aes_encrypt,
 819                         .cia_decrypt            = aes_decrypt
 820                 }
 821         }
 822 }, {
 823         .cra_name               = "__aes-aesni",
 824         .cra_driver_name        = "__driver-aes-aesni",
 825         .cra_priority           = 0,
 826         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 827         .cra_blocksize          = AES_BLOCK_SIZE,
 828         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
 829                                   AESNI_ALIGN - 1,
 830         .cra_alignmask          = 0,
 831         .cra_module             = THIS_MODULE,
 832         .cra_u  = {
 833                 .cipher = {
 834                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 835                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 836                         .cia_setkey             = aes_set_key,
 837                         .cia_encrypt            = __aes_encrypt,
 838                         .cia_decrypt            = __aes_decrypt
 839                 }
 840         }
 841 }, {
 842         .cra_name               = "__ecb-aes-aesni",
 843         .cra_driver_name        = "__driver-ecb-aes-aesni",
 844         .cra_priority           = 0,
 845         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 846         .cra_blocksize          = AES_BLOCK_SIZE,
 847         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
 848                                   AESNI_ALIGN - 1,
 849         .cra_alignmask          = 0,
 850         .cra_type               = &crypto_blkcipher_type,
 851         .cra_module             = THIS_MODULE,
 852         .cra_u = {
 853                 .blkcipher = {
 854                         .min_keysize    = AES_MIN_KEY_SIZE,
 855                         .max_keysize    = AES_MAX_KEY_SIZE,
 856                         .setkey         = aes_set_key,
 857                         .encrypt        = ecb_encrypt,
 858                         .decrypt        = ecb_decrypt,
 859                 },
 860         },
 861 }, {
 862         .cra_name               = "__cbc-aes-aesni",
 863         .cra_driver_name        = "__driver-cbc-aes-aesni",
 864         .cra_priority           = 0,
 865         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 866         .cra_blocksize          = AES_BLOCK_SIZE,
 867         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
 868                                   AESNI_ALIGN - 1,
 869         .cra_alignmask          = 0,
 870         .cra_type               = &crypto_blkcipher_type,
 871         .cra_module             = THIS_MODULE,
 872         .cra_u = {
 873                 .blkcipher = {
 874                         .min_keysize    = AES_MIN_KEY_SIZE,
 875                         .max_keysize    = AES_MAX_KEY_SIZE,
 876                         .setkey         = aes_set_key,
 877                         .encrypt        = cbc_encrypt,
 878                         .decrypt        = cbc_decrypt,
 879                 },
 880         },
 881 }, {
 882         .cra_name               = "ecb(aes)",
 883         .cra_driver_name        = "ecb-aes-aesni",
 884         .cra_priority           = 400,
 885         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 886         .cra_blocksize          = AES_BLOCK_SIZE,
 887         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 888         .cra_alignmask          = 0,
 889         .cra_type               = &crypto_ablkcipher_type,
 890         .cra_module             = THIS_MODULE,
 891         .cra_init               = ablk_ecb_init,
 892         .cra_exit               = ablk_exit,
 893         .cra_u = {
 894                 .ablkcipher = {
 895                         .min_keysize    = AES_MIN_KEY_SIZE,
 896                         .max_keysize    = AES_MAX_KEY_SIZE,
 897                         .setkey         = ablk_set_key,
 898                         .encrypt        = ablk_encrypt,
 899                         .decrypt        = ablk_decrypt,
 900                 },
 901         },
 902 }, {
 903         .cra_name               = "cbc(aes)",
 904         .cra_driver_name        = "cbc-aes-aesni",
 905         .cra_priority           = 400,
 906         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 907         .cra_blocksize          = AES_BLOCK_SIZE,
 908         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 909         .cra_alignmask          = 0,
 910         .cra_type               = &crypto_ablkcipher_type,
 911         .cra_module             = THIS_MODULE,
 912         .cra_init               = ablk_cbc_init,
 913         .cra_exit               = ablk_exit,
 914         .cra_u = {
 915                 .ablkcipher = {
 916                         .min_keysize    = AES_MIN_KEY_SIZE,
 917                         .max_keysize    = AES_MAX_KEY_SIZE,
 918                         .ivsize         = AES_BLOCK_SIZE,
 919                         .setkey         = ablk_set_key,
 920                         .encrypt        = ablk_encrypt,
 921                         .decrypt        = ablk_decrypt,
 922                 },
 923         },
 924 #ifdef CONFIG_X86_64
 925 }, {
 926         .cra_name               = "__ctr-aes-aesni",
 927         .cra_driver_name        = "__driver-ctr-aes-aesni",
 928         .cra_priority           = 0,
 929         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 930         .cra_blocksize          = 1,
 931         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
 932                                   AESNI_ALIGN - 1,
 933         .cra_alignmask          = 0,
 934         .cra_type               = &crypto_blkcipher_type,
 935         .cra_module             = THIS_MODULE,
 936         .cra_u = {
 937                 .blkcipher = {
 938                         .min_keysize    = AES_MIN_KEY_SIZE,
 939                         .max_keysize    = AES_MAX_KEY_SIZE,
 940                         .ivsize         = AES_BLOCK_SIZE,
 941                         .setkey         = aes_set_key,
 942                         .encrypt        = ctr_crypt,
 943                         .decrypt        = ctr_crypt,
 944                 },
 945         },
 946 }, {
 947         .cra_name               = "ctr(aes)",
 948         .cra_driver_name        = "ctr-aes-aesni",
 949         .cra_priority           = 400,
 950         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 951         .cra_blocksize          = 1,
 952         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 953         .cra_alignmask          = 0,
 954         .cra_type               = &crypto_ablkcipher_type,
 955         .cra_module             = THIS_MODULE,
 956         .cra_init               = ablk_ctr_init,
 957         .cra_exit               = ablk_exit,
 958         .cra_u = {
 959                 .ablkcipher = {
 960                         .min_keysize    = AES_MIN_KEY_SIZE,
 961                         .max_keysize    = AES_MAX_KEY_SIZE,
 962                         .ivsize         = AES_BLOCK_SIZE,
 963                         .setkey         = ablk_set_key,
 964                         .encrypt        = ablk_encrypt,
 965                         .decrypt        = ablk_encrypt,
 966                         .geniv          = "chainiv",
 967                 },
 968         },
 969 }, {
 970         .cra_name               = "__gcm-aes-aesni",
 971         .cra_driver_name        = "__driver-gcm-aes-aesni",
 972         .cra_priority           = 0,
 973         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
 974         .cra_blocksize          = 1,
 975         .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx) +
 976                                   AESNI_ALIGN,
 977         .cra_alignmask          = 0,
 978         .cra_type               = &crypto_aead_type,
 979         .cra_module             = THIS_MODULE,
 980         .cra_u = {
 981                 .aead = {
 982                         .encrypt        = __driver_rfc4106_encrypt,
 983                         .decrypt        = __driver_rfc4106_decrypt,
 984                 },
 985         },
 986 }, {
 987         .cra_name               = "rfc4106(gcm(aes))",
 988         .cra_driver_name        = "rfc4106-gcm-aesni",
 989         .cra_priority           = 400,
 990         .cra_flags              = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
 991         .cra_blocksize          = 1,
 992         .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx) +
 993                                   AESNI_ALIGN,
 994         .cra_alignmask          = 0,
 995         .cra_type               = &crypto_nivaead_type,
 996         .cra_module             = THIS_MODULE,
 997         .cra_init               = rfc4106_init,
 998         .cra_exit               = rfc4106_exit,
 999         .cra_u = {
1000                 .aead = {
1001                         .setkey         = rfc4106_set_key,
1002                         .setauthsize    = rfc4106_set_authsize,
1003                         .encrypt        = rfc4106_encrypt,
1004                         .decrypt        = rfc4106_decrypt,
1005                         .geniv          = "seqiv",
1006                         .ivsize         = 8,
1007                         .maxauthsize    = 16,
1008                 },
1009         },
1010 #ifdef HAS_CTR
1011 }, {
1012         .cra_name               = "rfc3686(ctr(aes))",
1013         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
1014         .cra_priority           = 400,
1015         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1016         .cra_blocksize          = 1,
1017         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1018         .cra_alignmask          = 0,
1019         .cra_type               = &crypto_ablkcipher_type,
1020         .cra_module             = THIS_MODULE,
1021         .cra_init               = ablk_rfc3686_ctr_init,
1022         .cra_exit               = ablk_exit,
1023         .cra_u = {
1024                 .ablkcipher = {
1025                         .min_keysize = AES_MIN_KEY_SIZE +
1026                                        CTR_RFC3686_NONCE_SIZE,
1027                         .max_keysize = AES_MAX_KEY_SIZE +
1028                                        CTR_RFC3686_NONCE_SIZE,
1029                         .ivsize      = CTR_RFC3686_IV_SIZE,
1030                         .setkey      = ablk_set_key,
1031                         .encrypt     = ablk_encrypt,
1032                         .decrypt     = ablk_decrypt,
1033                         .geniv       = "seqiv",
1034                 },
1035         },
1036 #endif
1037 #endif
1038 #ifdef HAS_LRW
1039 }, {
1040         .cra_name               = "lrw(aes)",
1041         .cra_driver_name        = "lrw-aes-aesni",
1042         .cra_priority           = 400,
1043         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1044         .cra_blocksize          = AES_BLOCK_SIZE,
1045         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1046         .cra_alignmask          = 0,
1047         .cra_type               = &crypto_ablkcipher_type,
1048         .cra_module             = THIS_MODULE,
1049         .cra_init               = ablk_lrw_init,
1050         .cra_exit               = ablk_exit,
1051         .cra_u = {
1052                 .ablkcipher = {
1053                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1054                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1055                         .ivsize         = AES_BLOCK_SIZE,
1056                         .setkey         = ablk_set_key,
1057                         .encrypt        = ablk_encrypt,
1058                         .decrypt        = ablk_decrypt,
1059                 },
1060         },
1061 #endif
1062 #ifdef HAS_PCBC
1063 }, {
1064         .cra_name               = "pcbc(aes)",
1065         .cra_driver_name        = "pcbc-aes-aesni",
1066         .cra_priority           = 400,
1067         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1068         .cra_blocksize          = AES_BLOCK_SIZE,
1069         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1070         .cra_alignmask          = 0,
1071         .cra_type               = &crypto_ablkcipher_type,
1072         .cra_module             = THIS_MODULE,
1073         .cra_init               = ablk_pcbc_init,
1074         .cra_exit               = ablk_exit,
1075         .cra_u = {
1076                 .ablkcipher = {
1077                         .min_keysize    = AES_MIN_KEY_SIZE,
1078                         .max_keysize    = AES_MAX_KEY_SIZE,
1079                         .ivsize         = AES_BLOCK_SIZE,
1080                         .setkey         = ablk_set_key,
1081                         .encrypt        = ablk_encrypt,
1082                         .decrypt        = ablk_decrypt,
1083                 },
1084         },
1085 #endif
1086 #ifdef HAS_XTS
1087 }, {
1088         .cra_name               = "xts(aes)",
1089         .cra_driver_name        = "xts-aes-aesni",
1090         .cra_priority           = 400,
1091         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1092         .cra_blocksize          = AES_BLOCK_SIZE,
1093         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1094         .cra_alignmask          = 0,
1095         .cra_type               = &crypto_ablkcipher_type,
1096         .cra_module             = THIS_MODULE,
1097         .cra_init               = ablk_xts_init,
1098         .cra_exit               = ablk_exit,
1099         .cra_u = {
1100                 .ablkcipher = {
1101                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1102                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1103                         .ivsize         = AES_BLOCK_SIZE,
1104                         .setkey         = ablk_set_key,
1105                         .encrypt        = ablk_encrypt,
1106                         .decrypt        = ablk_decrypt,
1107                 },
1108         },
1109 #endif
1110 } };
1111
1112
1113 static const struct x86_cpu_id aesni_cpu_id[] = {
1114         X86_FEATURE_MATCH(X86_FEATURE_AES),
1115         {}
1116 };
1117 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1118
1119 static int __init aesni_init(void)
1120 {
1121         int err, i;
1122
1123         if (!x86_match_cpu(aesni_cpu_id))
1124                 return -ENODEV;
1125
1126         err = crypto_fpu_init();
1127         if (err)
1128                 return err;
1129
1130         for (i = 0; i < ARRAY_SIZE(aesni_algs); i++)
1131                 INIT_LIST_HEAD(&aesni_algs[i].cra_list);
1132
1133         return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1134 }
1135
1136 static void __exit aesni_exit(void)
1137 {
1138         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1139
1140         crypto_fpu_exit();
1141 }
1142
1143 module_init(aesni_init);
1144 module_exit(aesni_exit);
1145
1146 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1147 MODULE_LICENSE("GPL");
1148 MODULE_ALIAS("aes");