arch/x86/crypto/aesni-intel_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Support for Intel AES-NI instructions. This file contains glue
   4  * code, the real AES implementation is in intel-aes_asm.S.
   5  *
   6  * Copyright (C) 2008, Intel Corp.
   7  *    Author: Huang Ying <ying.huang@intel.com>
   8  *
   9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  10  * interface for 64-bit kernels.
  11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  13  *             Tadeusz Struk (tadeusz.struk@intel.com)
  14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  15  *    Copyright (c) 2010, Intel Corporation.
  16  */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/types.h>
  20 #include <linux/module.h>
  21 #include <linux/err.h>
  22 #include <crypto/algapi.h>
  23 #include <crypto/aes.h>
  24 #include <crypto/ctr.h>
  25 #include <crypto/b128ops.h>
  26 #include <crypto/gcm.h>
  27 #include <crypto/xts.h>
  28 #include <asm/cpu_device_id.h>
  29 #include <asm/simd.h>
  30 #include <crypto/scatterwalk.h>
  31 #include <crypto/internal/aead.h>
  32 #include <crypto/internal/simd.h>
  33 #include <crypto/internal/skcipher.h>
  34 #include <linux/jump_label.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/spinlock.h>
  37 #include <linux/static_call.h>
  38
  39
  40 #define AESNI_ALIGN     16
  41 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  42 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  43 #define RFC4106_HASH_SUBKEY_SIZE 16
  44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  47
  48 /* This data is stored at the end of the crypto_tfm struct.
  49  * It's a type of per "session" data storage location.
  50  * This needs to be 16 byte aligned.
  51  */
  52 struct aesni_rfc4106_gcm_ctx {
  53         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  54         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  55         u8 nonce[4];
  56 };
  57
  58 struct generic_gcmaes_ctx {
  59         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  60         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  61 };
  62
  63 struct aesni_xts_ctx {
  64         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  65         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  66 };
  67
  68 #define GCM_BLOCK_LEN 16
  69
  70 struct gcm_context_data {
  71         /* init, update and finalize context data */
  72         u8 aad_hash[GCM_BLOCK_LEN];
  73         u64 aad_length;
  74         u64 in_length;
  75         u8 partial_block_enc_key[GCM_BLOCK_LEN];
  76         u8 orig_IV[GCM_BLOCK_LEN];
  77         u8 current_counter[GCM_BLOCK_LEN];
  78         u64 partial_block_len;
  79         u64 unused;
  80         u8 hash_keys[GCM_BLOCK_LEN * 16];
  81 };
  82
  83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  84                              unsigned int key_len);
  85 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
  86 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
  87 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  88                               const u8 *in, unsigned int len);
  89 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  90                               const u8 *in, unsigned int len);
  91 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  92                               const u8 *in, unsigned int len, u8 *iv);
  93 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  94                               const u8 *in, unsigned int len, u8 *iv);
  95 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  96                                   const u8 *in, unsigned int len, u8 *iv);
  97 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  98                                   const u8 *in, unsigned int len, u8 *iv);
  99
 100 #define AVX_GEN2_OPTSIZE 640
 101 #define AVX_GEN4_OPTSIZE 4096
 102
 103 asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out,
 104                                   const u8 *in, unsigned int len, u8 *iv);
 105
 106 asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out,
 107                                   const u8 *in, unsigned int len, u8 *iv);
 108
 109 #ifdef CONFIG_X86_64
 110
 111 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 112                               const u8 *in, unsigned int len, u8 *iv);
 113 DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc);
 114
 115 /* Scatter / Gather routines, with args similar to above */
 116 asmlinkage void aesni_gcm_init(void *ctx,
 117                                struct gcm_context_data *gdata,
 118                                u8 *iv,
 119                                u8 *hash_subkey, const u8 *aad,
 120                                unsigned long aad_len);
 121 asmlinkage void aesni_gcm_enc_update(void *ctx,
 122                                      struct gcm_context_data *gdata, u8 *out,
 123                                      const u8 *in, unsigned long plaintext_len);
 124 asmlinkage void aesni_gcm_dec_update(void *ctx,
 125                                      struct gcm_context_data *gdata, u8 *out,
 126                                      const u8 *in,
 127                                      unsigned long ciphertext_len);
 128 asmlinkage void aesni_gcm_finalize(void *ctx,
 129                                    struct gcm_context_data *gdata,
 130                                    u8 *auth_tag, unsigned long auth_tag_len);
 131
 132 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 133                 void *keys, u8 *out, unsigned int num_bytes);
 134 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 135                 void *keys, u8 *out, unsigned int num_bytes);
 136 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 137                 void *keys, u8 *out, unsigned int num_bytes);
 138
 139
 140 asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv,
 141         const void *keys, u8 *out, unsigned int num_bytes,
 142         unsigned int byte_ctr);
 143
 144 asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv,
 145         const void *keys, u8 *out, unsigned int num_bytes,
 146         unsigned int byte_ctr);
 147
 148 asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv,
 149         const void *keys, u8 *out, unsigned int num_bytes,
 150         unsigned int byte_ctr);
 151
 152 /*
 153  * asmlinkage void aesni_gcm_init_avx_gen2()
 154  * gcm_data *my_ctx_data, context data
 155  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 156  */
 157 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
 158                                         struct gcm_context_data *gdata,
 159                                         u8 *iv,
 160                                         u8 *hash_subkey,
 161                                         const u8 *aad,
 162                                         unsigned long aad_len);
 163
 164 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
 165                                      struct gcm_context_data *gdata, u8 *out,
 166                                      const u8 *in, unsigned long plaintext_len);
 167 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
 168                                      struct gcm_context_data *gdata, u8 *out,
 169                                      const u8 *in,
 170                                      unsigned long ciphertext_len);
 171 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
 172                                    struct gcm_context_data *gdata,
 173                                    u8 *auth_tag, unsigned long auth_tag_len);
 174
 175 /*
 176  * asmlinkage void aesni_gcm_init_avx_gen4()
 177  * gcm_data *my_ctx_data, context data
 178  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 179  */
 180 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
 181                                         struct gcm_context_data *gdata,
 182                                         u8 *iv,
 183                                         u8 *hash_subkey,
 184                                         const u8 *aad,
 185                                         unsigned long aad_len);
 186
 187 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
 188                                      struct gcm_context_data *gdata, u8 *out,
 189                                      const u8 *in, unsigned long plaintext_len);
 190 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
 191                                      struct gcm_context_data *gdata, u8 *out,
 192                                      const u8 *in,
 193                                      unsigned long ciphertext_len);
 194 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
 195                                    struct gcm_context_data *gdata,
 196                                    u8 *auth_tag, unsigned long auth_tag_len);
 197
 198 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx);
 199 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2);
 200
 201 static inline struct
 202 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 203 {
 204         unsigned long align = AESNI_ALIGN;
 205
 206         if (align <= crypto_tfm_ctx_alignment())
 207                 align = 1;
 208         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 209 }
 210
 211 static inline struct
 212 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 213 {
 214         unsigned long align = AESNI_ALIGN;
 215
 216         if (align <= crypto_tfm_ctx_alignment())
 217                 align = 1;
 218         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 219 }
 220 #endif
 221
 222 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 223 {
 224         unsigned long addr = (unsigned long)raw_ctx;
 225         unsigned long align = AESNI_ALIGN;
 226
 227         if (align <= crypto_tfm_ctx_alignment())
 228                 align = 1;
 229         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 230 }
 231
 232 static int aes_set_key_common(struct crypto_aes_ctx *ctx,
 233                               const u8 *in_key, unsigned int key_len)
 234 {
 235         int err;
 236
 237         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 238             key_len != AES_KEYSIZE_256)
 239                 return -EINVAL;
 240
 241         if (!crypto_simd_usable())
 242                 err = aes_expandkey(ctx, in_key, key_len);
 243         else {
 244                 kernel_fpu_begin();
 245                 err = aesni_set_key(ctx, in_key, key_len);
 246                 kernel_fpu_end();
 247         }
 248
 249         return err;
 250 }
 251
 252 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 253                        unsigned int key_len)
 254 {
 255         return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key,
 256                                   key_len);
 257 }
 258
 259 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 260 {
 261         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 262
 263         if (!crypto_simd_usable()) {
 264                 aes_encrypt(ctx, dst, src);
 265         } else {
 266                 kernel_fpu_begin();
 267                 aesni_enc(ctx, dst, src);
 268                 kernel_fpu_end();
 269         }
 270 }
 271
 272 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 273 {
 274         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 275
 276         if (!crypto_simd_usable()) {
 277                 aes_decrypt(ctx, dst, src);
 278         } else {
 279                 kernel_fpu_begin();
 280                 aesni_dec(ctx, dst, src);
 281                 kernel_fpu_end();
 282         }
 283 }
 284
 285 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 286                                  unsigned int len)
 287 {
 288         return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
 289 }
 290
 291 static int ecb_encrypt(struct skcipher_request *req)
 292 {
 293         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 294         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 295         struct skcipher_walk walk;
 296         unsigned int nbytes;
 297         int err;
 298
 299         err = skcipher_walk_virt(&walk, req, false);
 300
 301         while ((nbytes = walk.nbytes)) {
 302                 kernel_fpu_begin();
 303                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 304                               nbytes & AES_BLOCK_MASK);
 305                 kernel_fpu_end();
 306                 nbytes &= AES_BLOCK_SIZE - 1;
 307                 err = skcipher_walk_done(&walk, nbytes);
 308         }
 309
 310         return err;
 311 }
 312
 313 static int ecb_decrypt(struct skcipher_request *req)
 314 {
 315         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 316         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 317         struct skcipher_walk walk;
 318         unsigned int nbytes;
 319         int err;
 320
 321         err = skcipher_walk_virt(&walk, req, false);
 322
 323         while ((nbytes = walk.nbytes)) {
 324                 kernel_fpu_begin();
 325                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 326                               nbytes & AES_BLOCK_MASK);
 327                 kernel_fpu_end();
 328                 nbytes &= AES_BLOCK_SIZE - 1;
 329                 err = skcipher_walk_done(&walk, nbytes);
 330         }
 331
 332         return err;
 333 }
 334
 335 static int cbc_encrypt(struct skcipher_request *req)
 336 {
 337         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 338         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 339         struct skcipher_walk walk;
 340         unsigned int nbytes;
 341         int err;
 342
 343         err = skcipher_walk_virt(&walk, req, false);
 344
 345         while ((nbytes = walk.nbytes)) {
 346                 kernel_fpu_begin();
 347                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 348                               nbytes & AES_BLOCK_MASK, walk.iv);
 349                 kernel_fpu_end();
 350                 nbytes &= AES_BLOCK_SIZE - 1;
 351                 err = skcipher_walk_done(&walk, nbytes);
 352         }
 353
 354         return err;
 355 }
 356
 357 static int cbc_decrypt(struct skcipher_request *req)
 358 {
 359         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 360         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 361         struct skcipher_walk walk;
 362         unsigned int nbytes;
 363         int err;
 364
 365         err = skcipher_walk_virt(&walk, req, false);
 366
 367         while ((nbytes = walk.nbytes)) {
 368                 kernel_fpu_begin();
 369                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 370                               nbytes & AES_BLOCK_MASK, walk.iv);
 371                 kernel_fpu_end();
 372                 nbytes &= AES_BLOCK_SIZE - 1;
 373                 err = skcipher_walk_done(&walk, nbytes);
 374         }
 375
 376         return err;
 377 }
 378
 379 static int cts_cbc_encrypt(struct skcipher_request *req)
 380 {
 381         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 382         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 383         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 384         struct scatterlist *src = req->src, *dst = req->dst;
 385         struct scatterlist sg_src[2], sg_dst[2];
 386         struct skcipher_request subreq;
 387         struct skcipher_walk walk;
 388         int err;
 389
 390         skcipher_request_set_tfm(&subreq, tfm);
 391         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 392                                       NULL, NULL);
 393
 394         if (req->cryptlen <= AES_BLOCK_SIZE) {
 395                 if (req->cryptlen < AES_BLOCK_SIZE)
 396                         return -EINVAL;
 397                 cbc_blocks = 1;
 398         }
 399
 400         if (cbc_blocks > 0) {
 401                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 402                                            cbc_blocks * AES_BLOCK_SIZE,
 403                                            req->iv);
 404
 405                 err = cbc_encrypt(&subreq);
 406                 if (err)
 407                         return err;
 408
 409                 if (req->cryptlen == AES_BLOCK_SIZE)
 410                         return 0;
 411
 412                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 413                 if (req->dst != req->src)
 414                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 415                                                subreq.cryptlen);
 416         }
 417
 418         /* handle ciphertext stealing */
 419         skcipher_request_set_crypt(&subreq, src, dst,
 420                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 421                                    req->iv);
 422
 423         err = skcipher_walk_virt(&walk, &subreq, false);
 424         if (err)
 425                 return err;
 426
 427         kernel_fpu_begin();
 428         aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 429                           walk.nbytes, walk.iv);
 430         kernel_fpu_end();
 431
 432         return skcipher_walk_done(&walk, 0);
 433 }
 434
 435 static int cts_cbc_decrypt(struct skcipher_request *req)
 436 {
 437         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 438         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 439         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 440         struct scatterlist *src = req->src, *dst = req->dst;
 441         struct scatterlist sg_src[2], sg_dst[2];
 442         struct skcipher_request subreq;
 443         struct skcipher_walk walk;
 444         int err;
 445
 446         skcipher_request_set_tfm(&subreq, tfm);
 447         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 448                                       NULL, NULL);
 449
 450         if (req->cryptlen <= AES_BLOCK_SIZE) {
 451                 if (req->cryptlen < AES_BLOCK_SIZE)
 452                         return -EINVAL;
 453                 cbc_blocks = 1;
 454         }
 455
 456         if (cbc_blocks > 0) {
 457                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 458                                            cbc_blocks * AES_BLOCK_SIZE,
 459                                            req->iv);
 460
 461                 err = cbc_decrypt(&subreq);
 462                 if (err)
 463                         return err;
 464
 465                 if (req->cryptlen == AES_BLOCK_SIZE)
 466                         return 0;
 467
 468                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 469                 if (req->dst != req->src)
 470                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 471                                                subreq.cryptlen);
 472         }
 473
 474         /* handle ciphertext stealing */
 475         skcipher_request_set_crypt(&subreq, src, dst,
 476                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 477                                    req->iv);
 478
 479         err = skcipher_walk_virt(&walk, &subreq, false);
 480         if (err)
 481                 return err;
 482
 483         kernel_fpu_begin();
 484         aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 485                           walk.nbytes, walk.iv);
 486         kernel_fpu_end();
 487
 488         return skcipher_walk_done(&walk, 0);
 489 }
 490
 491 #ifdef CONFIG_X86_64
 492 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 493                               const u8 *in, unsigned int len, u8 *iv)
 494 {
 495         /*
 496          * based on key length, override with the by8 version
 497          * of ctr mode encryption/decryption for improved performance
 498          * aes_set_key_common() ensures that key length is one of
 499          * {128,192,256}
 500          */
 501         if (ctx->key_length == AES_KEYSIZE_128)
 502                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 503         else if (ctx->key_length == AES_KEYSIZE_192)
 504                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 505         else
 506                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 507 }
 508
 509 static int ctr_crypt(struct skcipher_request *req)
 510 {
 511         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 512         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 513         u8 keystream[AES_BLOCK_SIZE];
 514         struct skcipher_walk walk;
 515         unsigned int nbytes;
 516         int err;
 517
 518         err = skcipher_walk_virt(&walk, req, false);
 519
 520         while ((nbytes = walk.nbytes) > 0) {
 521                 kernel_fpu_begin();
 522                 if (nbytes & AES_BLOCK_MASK)
 523                         static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr,
 524                                                        walk.src.virt.addr,
 525                                                        nbytes & AES_BLOCK_MASK,
 526                                                        walk.iv);
 527                 nbytes &= ~AES_BLOCK_MASK;
 528
 529                 if (walk.nbytes == walk.total && nbytes > 0) {
 530                         aesni_enc(ctx, keystream, walk.iv);
 531                         crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
 532                                        walk.src.virt.addr + walk.nbytes - nbytes,
 533                                        keystream, nbytes);
 534                         crypto_inc(walk.iv, AES_BLOCK_SIZE);
 535                         nbytes = 0;
 536                 }
 537                 kernel_fpu_end();
 538                 err = skcipher_walk_done(&walk, nbytes);
 539         }
 540         return err;
 541 }
 542
 543 static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 544                                    const u8 *in, unsigned int len, u8 *iv,
 545                                    unsigned int byte_ctr)
 546 {
 547         if (ctx->key_length == AES_KEYSIZE_128)
 548                 aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len,
 549                                          byte_ctr);
 550         else if (ctx->key_length == AES_KEYSIZE_192)
 551                 aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len,
 552                                          byte_ctr);
 553         else
 554                 aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len,
 555                                          byte_ctr);
 556 }
 557
 558 static int xctr_crypt(struct skcipher_request *req)
 559 {
 560         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 561         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 562         u8 keystream[AES_BLOCK_SIZE];
 563         struct skcipher_walk walk;
 564         unsigned int nbytes;
 565         unsigned int byte_ctr = 0;
 566         int err;
 567         __le32 block[AES_BLOCK_SIZE / sizeof(__le32)];
 568
 569         err = skcipher_walk_virt(&walk, req, false);
 570
 571         while ((nbytes = walk.nbytes) > 0) {
 572                 kernel_fpu_begin();
 573                 if (nbytes & AES_BLOCK_MASK)
 574                         aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr,
 575                                 walk.src.virt.addr, nbytes & AES_BLOCK_MASK,
 576                                 walk.iv, byte_ctr);
 577                 nbytes &= ~AES_BLOCK_MASK;
 578                 byte_ctr += walk.nbytes - nbytes;
 579
 580                 if (walk.nbytes == walk.total && nbytes > 0) {
 581                         memcpy(block, walk.iv, AES_BLOCK_SIZE);
 582                         block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE);
 583                         aesni_enc(ctx, keystream, (u8 *)block);
 584                         crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes -
 585                                        nbytes, walk.src.virt.addr + walk.nbytes
 586                                        - nbytes, keystream, nbytes);
 587                         byte_ctr += nbytes;
 588                         nbytes = 0;
 589                 }
 590                 kernel_fpu_end();
 591                 err = skcipher_walk_done(&walk, nbytes);
 592         }
 593         return err;
 594 }
 595
 596 static int
 597 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 598 {
 599         struct crypto_aes_ctx ctx;
 600         int ret;
 601
 602         ret = aes_expandkey(&ctx, key, key_len);
 603         if (ret)
 604                 return ret;
 605
 606         /* Clear the data in the hash sub key container to zero.*/
 607         /* We want to cipher all zeros to create the hash sub key. */
 608         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 609
 610         aes_encrypt(&ctx, hash_subkey, hash_subkey);
 611
 612         memzero_explicit(&ctx, sizeof(ctx));
 613         return 0;
 614 }
 615
 616 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 617                                   unsigned int key_len)
 618 {
 619         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 620
 621         if (key_len < 4)
 622                 return -EINVAL;
 623
 624         /*Account for 4 byte nonce at the end.*/
 625         key_len -= 4;
 626
 627         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 628
 629         return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
 630                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 631 }
 632
 633 /* This is the Integrity Check Value (aka the authentication tag) length and can
 634  * be 8, 12 or 16 bytes long. */
 635 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 636                                        unsigned int authsize)
 637 {
 638         switch (authsize) {
 639         case 8:
 640         case 12:
 641         case 16:
 642                 break;
 643         default:
 644                 return -EINVAL;
 645         }
 646
 647         return 0;
 648 }
 649
 650 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 651                                        unsigned int authsize)
 652 {
 653         switch (authsize) {
 654         case 4:
 655         case 8:
 656         case 12:
 657         case 13:
 658         case 14:
 659         case 15:
 660         case 16:
 661                 break;
 662         default:
 663                 return -EINVAL;
 664         }
 665
 666         return 0;
 667 }
 668
 669 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
 670                               unsigned int assoclen, u8 *hash_subkey,
 671                               u8 *iv, void *aes_ctx, u8 *auth_tag,
 672                               unsigned long auth_tag_len)
 673 {
 674         u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8);
 675         struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN);
 676         unsigned long left = req->cryptlen;
 677         struct scatter_walk assoc_sg_walk;
 678         struct skcipher_walk walk;
 679         bool do_avx, do_avx2;
 680         u8 *assocmem = NULL;
 681         u8 *assoc;
 682         int err;
 683
 684         if (!enc)
 685                 left -= auth_tag_len;
 686
 687         do_avx = (left >= AVX_GEN2_OPTSIZE);
 688         do_avx2 = (left >= AVX_GEN4_OPTSIZE);
 689
 690         /* Linearize assoc, if not already linear */
 691         if (req->src->length >= assoclen && req->src->length) {
 692                 scatterwalk_start(&assoc_sg_walk, req->src);
 693                 assoc = scatterwalk_map(&assoc_sg_walk);
 694         } else {
 695                 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 696                               GFP_KERNEL : GFP_ATOMIC;
 697
 698                 /* assoc can be any length, so must be on heap */
 699                 assocmem = kmalloc(assoclen, flags);
 700                 if (unlikely(!assocmem))
 701                         return -ENOMEM;
 702                 assoc = assocmem;
 703
 704                 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
 705         }
 706
 707         kernel_fpu_begin();
 708         if (static_branch_likely(&gcm_use_avx2) && do_avx2)
 709                 aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc,
 710                                         assoclen);
 711         else if (static_branch_likely(&gcm_use_avx) && do_avx)
 712                 aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc,
 713                                         assoclen);
 714         else
 715                 aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen);
 716         kernel_fpu_end();
 717
 718         if (!assocmem)
 719                 scatterwalk_unmap(assoc);
 720         else
 721                 kfree(assocmem);
 722
 723         err = enc ? skcipher_walk_aead_encrypt(&walk, req, false)
 724                   : skcipher_walk_aead_decrypt(&walk, req, false);
 725
 726         while (walk.nbytes > 0) {
 727                 kernel_fpu_begin();
 728                 if (static_branch_likely(&gcm_use_avx2) && do_avx2) {
 729                         if (enc)
 730                                 aesni_gcm_enc_update_avx_gen4(aes_ctx, data,
 731                                                               walk.dst.virt.addr,
 732                                                               walk.src.virt.addr,
 733                                                               walk.nbytes);
 734                         else
 735                                 aesni_gcm_dec_update_avx_gen4(aes_ctx, data,
 736                                                               walk.dst.virt.addr,
 737                                                               walk.src.virt.addr,
 738                                                               walk.nbytes);
 739                 } else if (static_branch_likely(&gcm_use_avx) && do_avx) {
 740                         if (enc)
 741                                 aesni_gcm_enc_update_avx_gen2(aes_ctx, data,
 742                                                               walk.dst.virt.addr,
 743                                                               walk.src.virt.addr,
 744                                                               walk.nbytes);
 745                         else
 746                                 aesni_gcm_dec_update_avx_gen2(aes_ctx, data,
 747                                                               walk.dst.virt.addr,
 748                                                               walk.src.virt.addr,
 749                                                               walk.nbytes);
 750                 } else if (enc) {
 751                         aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr,
 752                                              walk.src.virt.addr, walk.nbytes);
 753                 } else {
 754                         aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr,
 755                                              walk.src.virt.addr, walk.nbytes);
 756                 }
 757                 kernel_fpu_end();
 758
 759                 err = skcipher_walk_done(&walk, 0);
 760         }
 761
 762         if (err)
 763                 return err;
 764
 765         kernel_fpu_begin();
 766         if (static_branch_likely(&gcm_use_avx2) && do_avx2)
 767                 aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag,
 768                                             auth_tag_len);
 769         else if (static_branch_likely(&gcm_use_avx) && do_avx)
 770                 aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag,
 771                                             auth_tag_len);
 772         else
 773                 aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len);
 774         kernel_fpu_end();
 775
 776         return 0;
 777 }
 778
 779 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 780                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 781 {
 782         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 783         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 784         u8 auth_tag[16];
 785         int err;
 786
 787         err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx,
 788                                  auth_tag, auth_tag_len);
 789         if (err)
 790                 return err;
 791
 792         scatterwalk_map_and_copy(auth_tag, req->dst,
 793                                  req->assoclen + req->cryptlen,
 794                                  auth_tag_len, 1);
 795         return 0;
 796 }
 797
 798 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 799                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 800 {
 801         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 802         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 803         u8 auth_tag_msg[16];
 804         u8 auth_tag[16];
 805         int err;
 806
 807         err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx,
 808                                  auth_tag, auth_tag_len);
 809         if (err)
 810                 return err;
 811
 812         /* Copy out original auth_tag */
 813         scatterwalk_map_and_copy(auth_tag_msg, req->src,
 814                                  req->assoclen + req->cryptlen - auth_tag_len,
 815                                  auth_tag_len, 0);
 816
 817         /* Compare generated tag with passed in tag. */
 818         if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) {
 819                 memzero_explicit(auth_tag, sizeof(auth_tag));
 820                 return -EBADMSG;
 821         }
 822         return 0;
 823 }
 824
 825 static int helper_rfc4106_encrypt(struct aead_request *req)
 826 {
 827         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 828         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 829         void *aes_ctx = &(ctx->aes_key_expanded);
 830         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
 831         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
 832         unsigned int i;
 833         __be32 counter = cpu_to_be32(1);
 834
 835         /* Assuming we are supporting rfc4106 64-bit extended */
 836         /* sequence numbers We need to have the AAD length equal */
 837         /* to 16 or 20 bytes */
 838         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 839                 return -EINVAL;
 840
 841         /* IV below built */
 842         for (i = 0; i < 4; i++)
 843                 *(iv+i) = ctx->nonce[i];
 844         for (i = 0; i < 8; i++)
 845                 *(iv+4+i) = req->iv[i];
 846         *((__be32 *)(iv+12)) = counter;
 847
 848         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 849                               aes_ctx);
 850 }
 851
 852 static int helper_rfc4106_decrypt(struct aead_request *req)
 853 {
 854         __be32 counter = cpu_to_be32(1);
 855         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 856         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 857         void *aes_ctx = &(ctx->aes_key_expanded);
 858         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
 859         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
 860         unsigned int i;
 861
 862         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 863                 return -EINVAL;
 864
 865         /* Assuming we are supporting rfc4106 64-bit extended */
 866         /* sequence numbers We need to have the AAD length */
 867         /* equal to 16 or 20 bytes */
 868
 869         /* IV below built */
 870         for (i = 0; i < 4; i++)
 871                 *(iv+i) = ctx->nonce[i];
 872         for (i = 0; i < 8; i++)
 873                 *(iv+4+i) = req->iv[i];
 874         *((__be32 *)(iv+12)) = counter;
 875
 876         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 877                               aes_ctx);
 878 }
 879 #endif
 880
 881 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 882                             unsigned int keylen)
 883 {
 884         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 885         int err;
 886
 887         err = xts_verify_key(tfm, key, keylen);
 888         if (err)
 889                 return err;
 890
 891         keylen /= 2;
 892
 893         /* first half of xts-key is for crypt */
 894         err = aes_set_key_common(aes_ctx(ctx->raw_crypt_ctx), key, keylen);
 895         if (err)
 896                 return err;
 897
 898         /* second half of xts-key is for tweak */
 899         return aes_set_key_common(aes_ctx(ctx->raw_tweak_ctx), key + keylen,
 900                                   keylen);
 901 }
 902
 903 static int xts_crypt(struct skcipher_request *req, bool encrypt)
 904 {
 905         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 906         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 907         int tail = req->cryptlen % AES_BLOCK_SIZE;
 908         struct skcipher_request subreq;
 909         struct skcipher_walk walk;
 910         int err;
 911
 912         if (req->cryptlen < AES_BLOCK_SIZE)
 913                 return -EINVAL;
 914
 915         err = skcipher_walk_virt(&walk, req, false);
 916         if (!walk.nbytes)
 917                 return err;
 918
 919         if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
 920                 int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 921
 922                 skcipher_walk_abort(&walk);
 923
 924                 skcipher_request_set_tfm(&subreq, tfm);
 925                 skcipher_request_set_callback(&subreq,
 926                                               skcipher_request_flags(req),
 927                                               NULL, NULL);
 928                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 929                                            blocks * AES_BLOCK_SIZE, req->iv);
 930                 req = &subreq;
 931
 932                 err = skcipher_walk_virt(&walk, req, false);
 933                 if (!walk.nbytes)
 934                         return err;
 935         } else {
 936                 tail = 0;
 937         }
 938
 939         kernel_fpu_begin();
 940
 941         /* calculate first value of T */
 942         aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv);
 943
 944         while (walk.nbytes > 0) {
 945                 int nbytes = walk.nbytes;
 946
 947                 if (nbytes < walk.total)
 948                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 949
 950                 if (encrypt)
 951                         aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
 952                                           walk.dst.virt.addr, walk.src.virt.addr,
 953                                           nbytes, walk.iv);
 954                 else
 955                         aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
 956                                           walk.dst.virt.addr, walk.src.virt.addr,
 957                                           nbytes, walk.iv);
 958                 kernel_fpu_end();
 959
 960                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 961
 962                 if (walk.nbytes > 0)
 963                         kernel_fpu_begin();
 964         }
 965
 966         if (unlikely(tail > 0 && !err)) {
 967                 struct scatterlist sg_src[2], sg_dst[2];
 968                 struct scatterlist *src, *dst;
 969
 970                 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 971                 if (req->dst != req->src)
 972                         dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 973
 974                 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 975                                            req->iv);
 976
 977                 err = skcipher_walk_virt(&walk, &subreq, false);
 978                 if (err)
 979                         return err;
 980
 981                 kernel_fpu_begin();
 982                 if (encrypt)
 983                         aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
 984                                           walk.dst.virt.addr, walk.src.virt.addr,
 985                                           walk.nbytes, walk.iv);
 986                 else
 987                         aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
 988                                           walk.dst.virt.addr, walk.src.virt.addr,
 989                                           walk.nbytes, walk.iv);
 990                 kernel_fpu_end();
 991
 992                 err = skcipher_walk_done(&walk, 0);
 993         }
 994         return err;
 995 }
 996
 997 static int xts_encrypt(struct skcipher_request *req)
 998 {
 999         return xts_crypt(req, true);
1000 }
1001
1002 static int xts_decrypt(struct skcipher_request *req)
1003 {
1004         return xts_crypt(req, false);
1005 }
1006
1007 static struct crypto_alg aesni_cipher_alg = {
1008         .cra_name               = "aes",
1009         .cra_driver_name        = "aes-aesni",
1010         .cra_priority           = 300,
1011         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
1012         .cra_blocksize          = AES_BLOCK_SIZE,
1013         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1014         .cra_module             = THIS_MODULE,
1015         .cra_u  = {
1016                 .cipher = {
1017                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
1018                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
1019                         .cia_setkey             = aes_set_key,
1020                         .cia_encrypt            = aesni_encrypt,
1021                         .cia_decrypt            = aesni_decrypt
1022                 }
1023         }
1024 };
1025
1026 static struct skcipher_alg aesni_skciphers[] = {
1027         {
1028                 .base = {
1029                         .cra_name               = "__ecb(aes)",
1030                         .cra_driver_name        = "__ecb-aes-aesni",
1031                         .cra_priority           = 400,
1032                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1033                         .cra_blocksize          = AES_BLOCK_SIZE,
1034                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1035                         .cra_module             = THIS_MODULE,
1036                 },
1037                 .min_keysize    = AES_MIN_KEY_SIZE,
1038                 .max_keysize    = AES_MAX_KEY_SIZE,
1039                 .setkey         = aesni_skcipher_setkey,
1040                 .encrypt        = ecb_encrypt,
1041                 .decrypt        = ecb_decrypt,
1042         }, {
1043                 .base = {
1044                         .cra_name               = "__cbc(aes)",
1045                         .cra_driver_name        = "__cbc-aes-aesni",
1046                         .cra_priority           = 400,
1047                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1048                         .cra_blocksize          = AES_BLOCK_SIZE,
1049                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1050                         .cra_module             = THIS_MODULE,
1051                 },
1052                 .min_keysize    = AES_MIN_KEY_SIZE,
1053                 .max_keysize    = AES_MAX_KEY_SIZE,
1054                 .ivsize         = AES_BLOCK_SIZE,
1055                 .setkey         = aesni_skcipher_setkey,
1056                 .encrypt        = cbc_encrypt,
1057                 .decrypt        = cbc_decrypt,
1058         }, {
1059                 .base = {
1060                         .cra_name               = "__cts(cbc(aes))",
1061                         .cra_driver_name        = "__cts-cbc-aes-aesni",
1062                         .cra_priority           = 400,
1063                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1064                         .cra_blocksize          = AES_BLOCK_SIZE,
1065                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1066                         .cra_module             = THIS_MODULE,
1067                 },
1068                 .min_keysize    = AES_MIN_KEY_SIZE,
1069                 .max_keysize    = AES_MAX_KEY_SIZE,
1070                 .ivsize         = AES_BLOCK_SIZE,
1071                 .walksize       = 2 * AES_BLOCK_SIZE,
1072                 .setkey         = aesni_skcipher_setkey,
1073                 .encrypt        = cts_cbc_encrypt,
1074                 .decrypt        = cts_cbc_decrypt,
1075 #ifdef CONFIG_X86_64
1076         }, {
1077                 .base = {
1078                         .cra_name               = "__ctr(aes)",
1079                         .cra_driver_name        = "__ctr-aes-aesni",
1080                         .cra_priority           = 400,
1081                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1082                         .cra_blocksize          = 1,
1083                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1084                         .cra_module             = THIS_MODULE,
1085                 },
1086                 .min_keysize    = AES_MIN_KEY_SIZE,
1087                 .max_keysize    = AES_MAX_KEY_SIZE,
1088                 .ivsize         = AES_BLOCK_SIZE,
1089                 .chunksize      = AES_BLOCK_SIZE,
1090                 .setkey         = aesni_skcipher_setkey,
1091                 .encrypt        = ctr_crypt,
1092                 .decrypt        = ctr_crypt,
1093 #endif
1094         }, {
1095                 .base = {
1096                         .cra_name               = "__xts(aes)",
1097                         .cra_driver_name        = "__xts-aes-aesni",
1098                         .cra_priority           = 401,
1099                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1100                         .cra_blocksize          = AES_BLOCK_SIZE,
1101                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
1102                         .cra_module             = THIS_MODULE,
1103                 },
1104                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1105                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1106                 .ivsize         = AES_BLOCK_SIZE,
1107                 .walksize       = 2 * AES_BLOCK_SIZE,
1108                 .setkey         = xts_aesni_setkey,
1109                 .encrypt        = xts_encrypt,
1110                 .decrypt        = xts_decrypt,
1111         }
1112 };
1113
1114 static
1115 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1116
1117 #ifdef CONFIG_X86_64
1118 /*
1119  * XCTR does not have a non-AVX implementation, so it must be enabled
1120  * conditionally.
1121  */
1122 static struct skcipher_alg aesni_xctr = {
1123         .base = {
1124                 .cra_name               = "__xctr(aes)",
1125                 .cra_driver_name        = "__xctr-aes-aesni",
1126                 .cra_priority           = 400,
1127                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1128                 .cra_blocksize          = 1,
1129                 .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1130                 .cra_module             = THIS_MODULE,
1131         },
1132         .min_keysize    = AES_MIN_KEY_SIZE,
1133         .max_keysize    = AES_MAX_KEY_SIZE,
1134         .ivsize         = AES_BLOCK_SIZE,
1135         .chunksize      = AES_BLOCK_SIZE,
1136         .setkey         = aesni_skcipher_setkey,
1137         .encrypt        = xctr_crypt,
1138         .decrypt        = xctr_crypt,
1139 };
1140
1141 static struct simd_skcipher_alg *aesni_simd_xctr;
1142 #endif /* CONFIG_X86_64 */
1143
1144 #ifdef CONFIG_X86_64
1145 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1146                                   unsigned int key_len)
1147 {
1148         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1149
1150         return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
1151                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1152 }
1153
1154 static int generic_gcmaes_encrypt(struct aead_request *req)
1155 {
1156         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1157         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1158         void *aes_ctx = &(ctx->aes_key_expanded);
1159         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1160         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1161         __be32 counter = cpu_to_be32(1);
1162
1163         memcpy(iv, req->iv, 12);
1164         *((__be32 *)(iv+12)) = counter;
1165
1166         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1167                               aes_ctx);
1168 }
1169
1170 static int generic_gcmaes_decrypt(struct aead_request *req)
1171 {
1172         __be32 counter = cpu_to_be32(1);
1173         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1174         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1175         void *aes_ctx = &(ctx->aes_key_expanded);
1176         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1177         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1178
1179         memcpy(iv, req->iv, 12);
1180         *((__be32 *)(iv+12)) = counter;
1181
1182         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1183                               aes_ctx);
1184 }
1185
1186 static struct aead_alg aesni_aeads[] = { {
1187         .setkey                 = common_rfc4106_set_key,
1188         .setauthsize            = common_rfc4106_set_authsize,
1189         .encrypt                = helper_rfc4106_encrypt,
1190         .decrypt                = helper_rfc4106_decrypt,
1191         .ivsize                 = GCM_RFC4106_IV_SIZE,
1192         .maxauthsize            = 16,
1193         .base = {
1194                 .cra_name               = "__rfc4106(gcm(aes))",
1195                 .cra_driver_name        = "__rfc4106-gcm-aesni",
1196                 .cra_priority           = 400,
1197                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1198                 .cra_blocksize          = 1,
1199                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1200                 .cra_alignmask          = 0,
1201                 .cra_module             = THIS_MODULE,
1202         },
1203 }, {
1204         .setkey                 = generic_gcmaes_set_key,
1205         .setauthsize            = generic_gcmaes_set_authsize,
1206         .encrypt                = generic_gcmaes_encrypt,
1207         .decrypt                = generic_gcmaes_decrypt,
1208         .ivsize                 = GCM_AES_IV_SIZE,
1209         .maxauthsize            = 16,
1210         .base = {
1211                 .cra_name               = "__gcm(aes)",
1212                 .cra_driver_name        = "__generic-gcm-aesni",
1213                 .cra_priority           = 400,
1214                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1215                 .cra_blocksize          = 1,
1216                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1217                 .cra_alignmask          = 0,
1218                 .cra_module             = THIS_MODULE,
1219         },
1220 } };
1221 #else
1222 static struct aead_alg aesni_aeads[0];
1223 #endif
1224
1225 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1226
1227 static const struct x86_cpu_id aesni_cpu_id[] = {
1228         X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1229         {}
1230 };
1231 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1232
1233 static int __init aesni_init(void)
1234 {
1235         int err;
1236
1237         if (!x86_match_cpu(aesni_cpu_id))
1238                 return -ENODEV;
1239 #ifdef CONFIG_X86_64
1240         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1241                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1242                 static_branch_enable(&gcm_use_avx);
1243                 static_branch_enable(&gcm_use_avx2);
1244         } else
1245         if (boot_cpu_has(X86_FEATURE_AVX)) {
1246                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1247                 static_branch_enable(&gcm_use_avx);
1248         } else {
1249                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1250         }
1251         if (boot_cpu_has(X86_FEATURE_AVX)) {
1252                 /* optimize performance of ctr mode encryption transform */
1253                 static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
1254                 pr_info("AES CTR mode by8 optimization enabled\n");
1255         }
1256 #endif /* CONFIG_X86_64 */
1257
1258         err = crypto_register_alg(&aesni_cipher_alg);
1259         if (err)
1260                 return err;
1261
1262         err = simd_register_skciphers_compat(aesni_skciphers,
1263                                              ARRAY_SIZE(aesni_skciphers),
1264                                              aesni_simd_skciphers);
1265         if (err)
1266                 goto unregister_cipher;
1267
1268         err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1269                                          aesni_simd_aeads);
1270         if (err)
1271                 goto unregister_skciphers;
1272
1273 #ifdef CONFIG_X86_64
1274         if (boot_cpu_has(X86_FEATURE_AVX))
1275                 err = simd_register_skciphers_compat(&aesni_xctr, 1,
1276                                                      &aesni_simd_xctr);
1277         if (err)
1278                 goto unregister_aeads;
1279 #endif /* CONFIG_X86_64 */
1280
1281         return 0;
1282
1283 #ifdef CONFIG_X86_64
1284 unregister_aeads:
1285         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1286                                 aesni_simd_aeads);
1287 #endif /* CONFIG_X86_64 */
1288
1289 unregister_skciphers:
1290         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1291                                   aesni_simd_skciphers);
1292 unregister_cipher:
1293         crypto_unregister_alg(&aesni_cipher_alg);
1294         return err;
1295 }
1296
1297 static void __exit aesni_exit(void)
1298 {
1299         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1300                               aesni_simd_aeads);
1301         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1302                                   aesni_simd_skciphers);
1303         crypto_unregister_alg(&aesni_cipher_alg);
1304 #ifdef CONFIG_X86_64
1305         if (boot_cpu_has(X86_FEATURE_AVX))
1306                 simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr);
1307 #endif /* CONFIG_X86_64 */
1308 }
1309
1310 late_initcall(aesni_init);
1311 module_exit(aesni_exit);
1312
1313 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1314 MODULE_LICENSE("GPL");
1315 MODULE_ALIAS_CRYPTO("aes");