arch/arm64/crypto/aes-glue.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
   4  *
   5  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
   6  */
   7
   8 #include <asm/neon.h>
   9 #include <asm/hwcap.h>
  10 #include <asm/simd.h>
  11 #include <crypto/aes.h>
  12 #include <crypto/ctr.h>
  13 #include <crypto/sha2.h>
  14 #include <crypto/internal/hash.h>
  15 #include <crypto/internal/simd.h>
  16 #include <crypto/internal/skcipher.h>
  17 #include <crypto/scatterwalk.h>
  18 #include <linux/module.h>
  19 #include <linux/cpufeature.h>
  20 #include <crypto/xts.h>
  21
  22 #include "aes-ce-setkey.h"
  23
  24 #ifdef USE_V8_CRYPTO_EXTENSIONS
  25 #define MODE                    "ce"
  26 #define PRIO                    300
  27 #define aes_expandkey           ce_aes_expandkey
  28 #define aes_ecb_encrypt         ce_aes_ecb_encrypt
  29 #define aes_ecb_decrypt         ce_aes_ecb_decrypt
  30 #define aes_cbc_encrypt         ce_aes_cbc_encrypt
  31 #define aes_cbc_decrypt         ce_aes_cbc_decrypt
  32 #define aes_cbc_cts_encrypt     ce_aes_cbc_cts_encrypt
  33 #define aes_cbc_cts_decrypt     ce_aes_cbc_cts_decrypt
  34 #define aes_essiv_cbc_encrypt   ce_aes_essiv_cbc_encrypt
  35 #define aes_essiv_cbc_decrypt   ce_aes_essiv_cbc_decrypt
  36 #define aes_ctr_encrypt         ce_aes_ctr_encrypt
  37 #define aes_xts_encrypt         ce_aes_xts_encrypt
  38 #define aes_xts_decrypt         ce_aes_xts_decrypt
  39 #define aes_mac_update          ce_aes_mac_update
  40 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
  41 #else
  42 #define MODE                    "neon"
  43 #define PRIO                    200
  44 #define aes_ecb_encrypt         neon_aes_ecb_encrypt
  45 #define aes_ecb_decrypt         neon_aes_ecb_decrypt
  46 #define aes_cbc_encrypt         neon_aes_cbc_encrypt
  47 #define aes_cbc_decrypt         neon_aes_cbc_decrypt
  48 #define aes_cbc_cts_encrypt     neon_aes_cbc_cts_encrypt
  49 #define aes_cbc_cts_decrypt     neon_aes_cbc_cts_decrypt
  50 #define aes_essiv_cbc_encrypt   neon_aes_essiv_cbc_encrypt
  51 #define aes_essiv_cbc_decrypt   neon_aes_essiv_cbc_decrypt
  52 #define aes_ctr_encrypt         neon_aes_ctr_encrypt
  53 #define aes_xts_encrypt         neon_aes_xts_encrypt
  54 #define aes_xts_decrypt         neon_aes_xts_decrypt
  55 #define aes_mac_update          neon_aes_mac_update
  56 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
  57 #endif
  58 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
  59 MODULE_ALIAS_CRYPTO("ecb(aes)");
  60 MODULE_ALIAS_CRYPTO("cbc(aes)");
  61 MODULE_ALIAS_CRYPTO("ctr(aes)");
  62 MODULE_ALIAS_CRYPTO("xts(aes)");
  63 #endif
  64 MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
  65 MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");
  66 MODULE_ALIAS_CRYPTO("cmac(aes)");
  67 MODULE_ALIAS_CRYPTO("xcbc(aes)");
  68 MODULE_ALIAS_CRYPTO("cbcmac(aes)");
  69
  70 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  71 MODULE_LICENSE("GPL v2");
  72
  73 /* defined in aes-modes.S */
  74 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
  75                                 int rounds, int blocks);
  76 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
  77                                 int rounds, int blocks);
  78
  79 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
  80                                 int rounds, int blocks, u8 iv[]);
  81 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
  82                                 int rounds, int blocks, u8 iv[]);
  83
  84 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
  85                                 int rounds, int bytes, u8 const iv[]);
  86 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
  87                                 int rounds, int bytes, u8 const iv[]);
  88
  89 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
  90                                 int rounds, int bytes, u8 ctr[]);
  91
  92 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
  93                                 int rounds, int bytes, u32 const rk2[], u8 iv[],
  94                                 int first);
  95 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
  96                                 int rounds, int bytes, u32 const rk2[], u8 iv[],
  97                                 int first);
  98
  99 asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
 100                                       int rounds, int blocks, u8 iv[],
 101                                       u32 const rk2[]);
 102 asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
 103                                       int rounds, int blocks, u8 iv[],
 104                                       u32 const rk2[]);
 105
 106 asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds,
 107                               int blocks, u8 dg[], int enc_before,
 108                               int enc_after);
 109
 110 struct crypto_aes_xts_ctx {
 111         struct crypto_aes_ctx key1;
 112         struct crypto_aes_ctx __aligned(8) key2;
 113 };
 114
 115 struct crypto_aes_essiv_cbc_ctx {
 116         struct crypto_aes_ctx key1;
 117         struct crypto_aes_ctx __aligned(8) key2;
 118         struct crypto_shash *hash;
 119 };
 120
 121 struct mac_tfm_ctx {
 122         struct crypto_aes_ctx key;
 123         u8 __aligned(8) consts[];
 124 };
 125
 126 struct mac_desc_ctx {
 127         unsigned int len;
 128         u8 dg[AES_BLOCK_SIZE];
 129 };
 130
 131 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 132                                unsigned int key_len)
 133 {
 134         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 135
 136         return aes_expandkey(ctx, in_key, key_len);
 137 }
 138
 139 static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm,
 140                                       const u8 *in_key, unsigned int key_len)
 141 {
 142         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 143         int ret;
 144
 145         ret = xts_verify_key(tfm, in_key, key_len);
 146         if (ret)
 147                 return ret;
 148
 149         ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
 150         if (!ret)
 151                 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
 152                                     key_len / 2);
 153         return ret;
 154 }
 155
 156 static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm,
 157                                             const u8 *in_key,
 158                                             unsigned int key_len)
 159 {
 160         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 161         u8 digest[SHA256_DIGEST_SIZE];
 162         int ret;
 163
 164         ret = aes_expandkey(&ctx->key1, in_key, key_len);
 165         if (ret)
 166                 return ret;
 167
 168         crypto_shash_tfm_digest(ctx->hash, in_key, key_len, digest);
 169
 170         return aes_expandkey(&ctx->key2, digest, sizeof(digest));
 171 }
 172
 173 static int __maybe_unused ecb_encrypt(struct skcipher_request *req)
 174 {
 175         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 176         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 177         int err, rounds = 6 + ctx->key_length / 4;
 178         struct skcipher_walk walk;
 179         unsigned int blocks;
 180
 181         err = skcipher_walk_virt(&walk, req, false);
 182
 183         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 184                 kernel_neon_begin();
 185                 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 186                                 ctx->key_enc, rounds, blocks);
 187                 kernel_neon_end();
 188                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 189         }
 190         return err;
 191 }
 192
 193 static int __maybe_unused ecb_decrypt(struct skcipher_request *req)
 194 {
 195         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 196         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 197         int err, rounds = 6 + ctx->key_length / 4;
 198         struct skcipher_walk walk;
 199         unsigned int blocks;
 200
 201         err = skcipher_walk_virt(&walk, req, false);
 202
 203         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 204                 kernel_neon_begin();
 205                 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 206                                 ctx->key_dec, rounds, blocks);
 207                 kernel_neon_end();
 208                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 209         }
 210         return err;
 211 }
 212
 213 static int cbc_encrypt_walk(struct skcipher_request *req,
 214                             struct skcipher_walk *walk)
 215 {
 216         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 217         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 218         int err = 0, rounds = 6 + ctx->key_length / 4;
 219         unsigned int blocks;
 220
 221         while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
 222                 kernel_neon_begin();
 223                 aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
 224                                 ctx->key_enc, rounds, blocks, walk->iv);
 225                 kernel_neon_end();
 226                 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
 227         }
 228         return err;
 229 }
 230
 231 static int __maybe_unused cbc_encrypt(struct skcipher_request *req)
 232 {
 233         struct skcipher_walk walk;
 234         int err;
 235
 236         err = skcipher_walk_virt(&walk, req, false);
 237         if (err)
 238                 return err;
 239         return cbc_encrypt_walk(req, &walk);
 240 }
 241
 242 static int cbc_decrypt_walk(struct skcipher_request *req,
 243                             struct skcipher_walk *walk)
 244 {
 245         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 246         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 247         int err = 0, rounds = 6 + ctx->key_length / 4;
 248         unsigned int blocks;
 249
 250         while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
 251                 kernel_neon_begin();
 252                 aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
 253                                 ctx->key_dec, rounds, blocks, walk->iv);
 254                 kernel_neon_end();
 255                 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
 256         }
 257         return err;
 258 }
 259
 260 static int __maybe_unused cbc_decrypt(struct skcipher_request *req)
 261 {
 262         struct skcipher_walk walk;
 263         int err;
 264
 265         err = skcipher_walk_virt(&walk, req, false);
 266         if (err)
 267                 return err;
 268         return cbc_decrypt_walk(req, &walk);
 269 }
 270
 271 static int cts_cbc_encrypt(struct skcipher_request *req)
 272 {
 273         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 274         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 275         int err, rounds = 6 + ctx->key_length / 4;
 276         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 277         struct scatterlist *src = req->src, *dst = req->dst;
 278         struct scatterlist sg_src[2], sg_dst[2];
 279         struct skcipher_request subreq;
 280         struct skcipher_walk walk;
 281
 282         skcipher_request_set_tfm(&subreq, tfm);
 283         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 284                                       NULL, NULL);
 285
 286         if (req->cryptlen <= AES_BLOCK_SIZE) {
 287                 if (req->cryptlen < AES_BLOCK_SIZE)
 288                         return -EINVAL;
 289                 cbc_blocks = 1;
 290         }
 291
 292         if (cbc_blocks > 0) {
 293                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 294                                            cbc_blocks * AES_BLOCK_SIZE,
 295                                            req->iv);
 296
 297                 err = skcipher_walk_virt(&walk, &subreq, false) ?:
 298                       cbc_encrypt_walk(&subreq, &walk);
 299                 if (err)
 300                         return err;
 301
 302                 if (req->cryptlen == AES_BLOCK_SIZE)
 303                         return 0;
 304
 305                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 306                 if (req->dst != req->src)
 307                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 308                                                subreq.cryptlen);
 309         }
 310
 311         /* handle ciphertext stealing */
 312         skcipher_request_set_crypt(&subreq, src, dst,
 313                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 314                                    req->iv);
 315
 316         err = skcipher_walk_virt(&walk, &subreq, false);
 317         if (err)
 318                 return err;
 319
 320         kernel_neon_begin();
 321         aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 322                             ctx->key_enc, rounds, walk.nbytes, walk.iv);
 323         kernel_neon_end();
 324
 325         return skcipher_walk_done(&walk, 0);
 326 }
 327
 328 static int cts_cbc_decrypt(struct skcipher_request *req)
 329 {
 330         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 331         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 332         int err, rounds = 6 + ctx->key_length / 4;
 333         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 334         struct scatterlist *src = req->src, *dst = req->dst;
 335         struct scatterlist sg_src[2], sg_dst[2];
 336         struct skcipher_request subreq;
 337         struct skcipher_walk walk;
 338
 339         skcipher_request_set_tfm(&subreq, tfm);
 340         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 341                                       NULL, NULL);
 342
 343         if (req->cryptlen <= AES_BLOCK_SIZE) {
 344                 if (req->cryptlen < AES_BLOCK_SIZE)
 345                         return -EINVAL;
 346                 cbc_blocks = 1;
 347         }
 348
 349         if (cbc_blocks > 0) {
 350                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 351                                            cbc_blocks * AES_BLOCK_SIZE,
 352                                            req->iv);
 353
 354                 err = skcipher_walk_virt(&walk, &subreq, false) ?:
 355                       cbc_decrypt_walk(&subreq, &walk);
 356                 if (err)
 357                         return err;
 358
 359                 if (req->cryptlen == AES_BLOCK_SIZE)
 360                         return 0;
 361
 362                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 363                 if (req->dst != req->src)
 364                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 365                                                subreq.cryptlen);
 366         }
 367
 368         /* handle ciphertext stealing */
 369         skcipher_request_set_crypt(&subreq, src, dst,
 370                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 371                                    req->iv);
 372
 373         err = skcipher_walk_virt(&walk, &subreq, false);
 374         if (err)
 375                 return err;
 376
 377         kernel_neon_begin();
 378         aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 379                             ctx->key_dec, rounds, walk.nbytes, walk.iv);
 380         kernel_neon_end();
 381
 382         return skcipher_walk_done(&walk, 0);
 383 }
 384
 385 static int __maybe_unused essiv_cbc_init_tfm(struct crypto_skcipher *tfm)
 386 {
 387         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 388
 389         ctx->hash = crypto_alloc_shash("sha256", 0, 0);
 390
 391         return PTR_ERR_OR_ZERO(ctx->hash);
 392 }
 393
 394 static void __maybe_unused essiv_cbc_exit_tfm(struct crypto_skcipher *tfm)
 395 {
 396         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 397
 398         crypto_free_shash(ctx->hash);
 399 }
 400
 401 static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req)
 402 {
 403         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 404         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 405         int err, rounds = 6 + ctx->key1.key_length / 4;
 406         struct skcipher_walk walk;
 407         unsigned int blocks;
 408
 409         err = skcipher_walk_virt(&walk, req, false);
 410
 411         blocks = walk.nbytes / AES_BLOCK_SIZE;
 412         if (blocks) {
 413                 kernel_neon_begin();
 414                 aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 415                                       ctx->key1.key_enc, rounds, blocks,
 416                                       req->iv, ctx->key2.key_enc);
 417                 kernel_neon_end();
 418                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 419         }
 420         return err ?: cbc_encrypt_walk(req, &walk);
 421 }
 422
 423 static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req)
 424 {
 425         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 426         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 427         int err, rounds = 6 + ctx->key1.key_length / 4;
 428         struct skcipher_walk walk;
 429         unsigned int blocks;
 430
 431         err = skcipher_walk_virt(&walk, req, false);
 432
 433         blocks = walk.nbytes / AES_BLOCK_SIZE;
 434         if (blocks) {
 435                 kernel_neon_begin();
 436                 aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 437                                       ctx->key1.key_dec, rounds, blocks,
 438                                       req->iv, ctx->key2.key_enc);
 439                 kernel_neon_end();
 440                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 441         }
 442         return err ?: cbc_decrypt_walk(req, &walk);
 443 }
 444
 445 static int __maybe_unused ctr_encrypt(struct skcipher_request *req)
 446 {
 447         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 448         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 449         int err, rounds = 6 + ctx->key_length / 4;
 450         struct skcipher_walk walk;
 451
 452         err = skcipher_walk_virt(&walk, req, false);
 453
 454         while (walk.nbytes > 0) {
 455                 const u8 *src = walk.src.virt.addr;
 456                 unsigned int nbytes = walk.nbytes;
 457                 u8 *dst = walk.dst.virt.addr;
 458                 u8 buf[AES_BLOCK_SIZE];
 459
 460                 if (unlikely(nbytes < AES_BLOCK_SIZE))
 461                         src = dst = memcpy(buf + sizeof(buf) - nbytes,
 462                                            src, nbytes);
 463                 else if (nbytes < walk.total)
 464                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 465
 466                 kernel_neon_begin();
 467                 aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes,
 468                                 walk.iv);
 469                 kernel_neon_end();
 470
 471                 if (unlikely(nbytes < AES_BLOCK_SIZE))
 472                         memcpy(walk.dst.virt.addr,
 473                                buf + sizeof(buf) - nbytes, nbytes);
 474
 475                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 476         }
 477
 478         return err;
 479 }
 480
 481 static int __maybe_unused xts_encrypt(struct skcipher_request *req)
 482 {
 483         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 484         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 485         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 486         int tail = req->cryptlen % AES_BLOCK_SIZE;
 487         struct scatterlist sg_src[2], sg_dst[2];
 488         struct skcipher_request subreq;
 489         struct scatterlist *src, *dst;
 490         struct skcipher_walk walk;
 491
 492         if (req->cryptlen < AES_BLOCK_SIZE)
 493                 return -EINVAL;
 494
 495         err = skcipher_walk_virt(&walk, req, false);
 496
 497         if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
 498                 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
 499                                               AES_BLOCK_SIZE) - 2;
 500
 501                 skcipher_walk_abort(&walk);
 502
 503                 skcipher_request_set_tfm(&subreq, tfm);
 504                 skcipher_request_set_callback(&subreq,
 505                                               skcipher_request_flags(req),
 506                                               NULL, NULL);
 507                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 508                                            xts_blocks * AES_BLOCK_SIZE,
 509                                            req->iv);
 510                 req = &subreq;
 511                 err = skcipher_walk_virt(&walk, req, false);
 512         } else {
 513                 tail = 0;
 514         }
 515
 516         for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
 517                 int nbytes = walk.nbytes;
 518
 519                 if (walk.nbytes < walk.total)
 520                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 521
 522                 kernel_neon_begin();
 523                 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 524                                 ctx->key1.key_enc, rounds, nbytes,
 525                                 ctx->key2.key_enc, walk.iv, first);
 526                 kernel_neon_end();
 527                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 528         }
 529
 530         if (err || likely(!tail))
 531                 return err;
 532
 533         dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 534         if (req->dst != req->src)
 535                 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 536
 537         skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 538                                    req->iv);
 539
 540         err = skcipher_walk_virt(&walk, &subreq, false);
 541         if (err)
 542                 return err;
 543
 544         kernel_neon_begin();
 545         aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 546                         ctx->key1.key_enc, rounds, walk.nbytes,
 547                         ctx->key2.key_enc, walk.iv, first);
 548         kernel_neon_end();
 549
 550         return skcipher_walk_done(&walk, 0);
 551 }
 552
 553 static int __maybe_unused xts_decrypt(struct skcipher_request *req)
 554 {
 555         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 556         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 557         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 558         int tail = req->cryptlen % AES_BLOCK_SIZE;
 559         struct scatterlist sg_src[2], sg_dst[2];
 560         struct skcipher_request subreq;
 561         struct scatterlist *src, *dst;
 562         struct skcipher_walk walk;
 563
 564         if (req->cryptlen < AES_BLOCK_SIZE)
 565                 return -EINVAL;
 566
 567         err = skcipher_walk_virt(&walk, req, false);
 568
 569         if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
 570                 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
 571                                               AES_BLOCK_SIZE) - 2;
 572
 573                 skcipher_walk_abort(&walk);
 574
 575                 skcipher_request_set_tfm(&subreq, tfm);
 576                 skcipher_request_set_callback(&subreq,
 577                                               skcipher_request_flags(req),
 578                                               NULL, NULL);
 579                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 580                                            xts_blocks * AES_BLOCK_SIZE,
 581                                            req->iv);
 582                 req = &subreq;
 583                 err = skcipher_walk_virt(&walk, req, false);
 584         } else {
 585                 tail = 0;
 586         }
 587
 588         for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
 589                 int nbytes = walk.nbytes;
 590
 591                 if (walk.nbytes < walk.total)
 592                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 593
 594                 kernel_neon_begin();
 595                 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 596                                 ctx->key1.key_dec, rounds, nbytes,
 597                                 ctx->key2.key_enc, walk.iv, first);
 598                 kernel_neon_end();
 599                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 600         }
 601
 602         if (err || likely(!tail))
 603                 return err;
 604
 605         dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 606         if (req->dst != req->src)
 607                 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 608
 609         skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 610                                    req->iv);
 611
 612         err = skcipher_walk_virt(&walk, &subreq, false);
 613         if (err)
 614                 return err;
 615
 616
 617         kernel_neon_begin();
 618         aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 619                         ctx->key1.key_dec, rounds, walk.nbytes,
 620                         ctx->key2.key_enc, walk.iv, first);
 621         kernel_neon_end();
 622
 623         return skcipher_walk_done(&walk, 0);
 624 }
 625
 626 static struct skcipher_alg aes_algs[] = { {
 627 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
 628         .base = {
 629                 .cra_name               = "ecb(aes)",
 630                 .cra_driver_name        = "ecb-aes-" MODE,
 631                 .cra_priority           = PRIO,
 632                 .cra_blocksize          = AES_BLOCK_SIZE,
 633                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 634                 .cra_module             = THIS_MODULE,
 635         },
 636         .min_keysize    = AES_MIN_KEY_SIZE,
 637         .max_keysize    = AES_MAX_KEY_SIZE,
 638         .setkey         = skcipher_aes_setkey,
 639         .encrypt        = ecb_encrypt,
 640         .decrypt        = ecb_decrypt,
 641 }, {
 642         .base = {
 643                 .cra_name               = "cbc(aes)",
 644                 .cra_driver_name        = "cbc-aes-" MODE,
 645                 .cra_priority           = PRIO,
 646                 .cra_blocksize          = AES_BLOCK_SIZE,
 647                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 648                 .cra_module             = THIS_MODULE,
 649         },
 650         .min_keysize    = AES_MIN_KEY_SIZE,
 651         .max_keysize    = AES_MAX_KEY_SIZE,
 652         .ivsize         = AES_BLOCK_SIZE,
 653         .setkey         = skcipher_aes_setkey,
 654         .encrypt        = cbc_encrypt,
 655         .decrypt        = cbc_decrypt,
 656 }, {
 657         .base = {
 658                 .cra_name               = "ctr(aes)",
 659                 .cra_driver_name        = "ctr-aes-" MODE,
 660                 .cra_priority           = PRIO,
 661                 .cra_blocksize          = 1,
 662                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 663                 .cra_module             = THIS_MODULE,
 664         },
 665         .min_keysize    = AES_MIN_KEY_SIZE,
 666         .max_keysize    = AES_MAX_KEY_SIZE,
 667         .ivsize         = AES_BLOCK_SIZE,
 668         .chunksize      = AES_BLOCK_SIZE,
 669         .setkey         = skcipher_aes_setkey,
 670         .encrypt        = ctr_encrypt,
 671         .decrypt        = ctr_encrypt,
 672 }, {
 673         .base = {
 674                 .cra_name               = "xts(aes)",
 675                 .cra_driver_name        = "xts-aes-" MODE,
 676                 .cra_priority           = PRIO,
 677                 .cra_blocksize          = AES_BLOCK_SIZE,
 678                 .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
 679                 .cra_module             = THIS_MODULE,
 680         },
 681         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 682         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 683         .ivsize         = AES_BLOCK_SIZE,
 684         .walksize       = 2 * AES_BLOCK_SIZE,
 685         .setkey         = xts_set_key,
 686         .encrypt        = xts_encrypt,
 687         .decrypt        = xts_decrypt,
 688 }, {
 689 #endif
 690         .base = {
 691                 .cra_name               = "cts(cbc(aes))",
 692                 .cra_driver_name        = "cts-cbc-aes-" MODE,
 693                 .cra_priority           = PRIO,
 694                 .cra_blocksize          = AES_BLOCK_SIZE,
 695                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 696                 .cra_module             = THIS_MODULE,
 697         },
 698         .min_keysize    = AES_MIN_KEY_SIZE,
 699         .max_keysize    = AES_MAX_KEY_SIZE,
 700         .ivsize         = AES_BLOCK_SIZE,
 701         .walksize       = 2 * AES_BLOCK_SIZE,
 702         .setkey         = skcipher_aes_setkey,
 703         .encrypt        = cts_cbc_encrypt,
 704         .decrypt        = cts_cbc_decrypt,
 705 }, {
 706         .base = {
 707                 .cra_name               = "essiv(cbc(aes),sha256)",
 708                 .cra_driver_name        = "essiv-cbc-aes-sha256-" MODE,
 709                 .cra_priority           = PRIO + 1,
 710                 .cra_blocksize          = AES_BLOCK_SIZE,
 711                 .cra_ctxsize            = sizeof(struct crypto_aes_essiv_cbc_ctx),
 712                 .cra_module             = THIS_MODULE,
 713         },
 714         .min_keysize    = AES_MIN_KEY_SIZE,
 715         .max_keysize    = AES_MAX_KEY_SIZE,
 716         .ivsize         = AES_BLOCK_SIZE,
 717         .setkey         = essiv_cbc_set_key,
 718         .encrypt        = essiv_cbc_encrypt,
 719         .decrypt        = essiv_cbc_decrypt,
 720         .init           = essiv_cbc_init_tfm,
 721         .exit           = essiv_cbc_exit_tfm,
 722 } };
 723
 724 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
 725                          unsigned int key_len)
 726 {
 727         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
 728
 729         return aes_expandkey(&ctx->key, in_key, key_len);
 730 }
 731
 732 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
 733 {
 734         u64 a = be64_to_cpu(x->a);
 735         u64 b = be64_to_cpu(x->b);
 736
 737         y->a = cpu_to_be64((a << 1) | (b >> 63));
 738         y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
 739 }
 740
 741 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
 742                        unsigned int key_len)
 743 {
 744         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
 745         be128 *consts = (be128 *)ctx->consts;
 746         int rounds = 6 + key_len / 4;
 747         int err;
 748
 749         err = cbcmac_setkey(tfm, in_key, key_len);
 750         if (err)
 751                 return err;
 752
 753         /* encrypt the zero vector */
 754         kernel_neon_begin();
 755         aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc,
 756                         rounds, 1);
 757         kernel_neon_end();
 758
 759         cmac_gf128_mul_by_x(consts, consts);
 760         cmac_gf128_mul_by_x(consts + 1, consts);
 761
 762         return 0;
 763 }
 764
 765 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
 766                        unsigned int key_len)
 767 {
 768         static u8 const ks[3][AES_BLOCK_SIZE] = {
 769                 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
 770                 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
 771                 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
 772         };
 773
 774         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
 775         int rounds = 6 + key_len / 4;
 776         u8 key[AES_BLOCK_SIZE];
 777         int err;
 778
 779         err = cbcmac_setkey(tfm, in_key, key_len);
 780         if (err)
 781                 return err;
 782
 783         kernel_neon_begin();
 784         aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
 785         aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
 786         kernel_neon_end();
 787
 788         return cbcmac_setkey(tfm, key, sizeof(key));
 789 }
 790
 791 static int mac_init(struct shash_desc *desc)
 792 {
 793         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 794
 795         memset(ctx->dg, 0, AES_BLOCK_SIZE);
 796         ctx->len = 0;
 797
 798         return 0;
 799 }
 800
 801 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
 802                           u8 dg[], int enc_before, int enc_after)
 803 {
 804         int rounds = 6 + ctx->key_length / 4;
 805
 806         if (crypto_simd_usable()) {
 807                 int rem;
 808
 809                 do {
 810                         kernel_neon_begin();
 811                         rem = aes_mac_update(in, ctx->key_enc, rounds, blocks,
 812                                              dg, enc_before, enc_after);
 813                         kernel_neon_end();
 814                         in += (blocks - rem) * AES_BLOCK_SIZE;
 815                         blocks = rem;
 816                         enc_before = 0;
 817                 } while (blocks);
 818         } else {
 819                 if (enc_before)
 820                         aes_encrypt(ctx, dg, dg);
 821
 822                 while (blocks--) {
 823                         crypto_xor(dg, in, AES_BLOCK_SIZE);
 824                         in += AES_BLOCK_SIZE;
 825
 826                         if (blocks || enc_after)
 827                                 aes_encrypt(ctx, dg, dg);
 828                 }
 829         }
 830 }
 831
 832 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
 833 {
 834         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 835         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 836
 837         while (len > 0) {
 838                 unsigned int l;
 839
 840                 if ((ctx->len % AES_BLOCK_SIZE) == 0 &&
 841                     (ctx->len + len) > AES_BLOCK_SIZE) {
 842
 843                         int blocks = len / AES_BLOCK_SIZE;
 844
 845                         len %= AES_BLOCK_SIZE;
 846
 847                         mac_do_update(&tctx->key, p, blocks, ctx->dg,
 848                                       (ctx->len != 0), (len != 0));
 849
 850                         p += blocks * AES_BLOCK_SIZE;
 851
 852                         if (!len) {
 853                                 ctx->len = AES_BLOCK_SIZE;
 854                                 break;
 855                         }
 856                         ctx->len = 0;
 857                 }
 858
 859                 l = min(len, AES_BLOCK_SIZE - ctx->len);
 860
 861                 if (l <= AES_BLOCK_SIZE) {
 862                         crypto_xor(ctx->dg + ctx->len, p, l);
 863                         ctx->len += l;
 864                         len -= l;
 865                         p += l;
 866                 }
 867         }
 868
 869         return 0;
 870 }
 871
 872 static int cbcmac_final(struct shash_desc *desc, u8 *out)
 873 {
 874         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 875         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 876
 877         mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0);
 878
 879         memcpy(out, ctx->dg, AES_BLOCK_SIZE);
 880
 881         return 0;
 882 }
 883
 884 static int cmac_final(struct shash_desc *desc, u8 *out)
 885 {
 886         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 887         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 888         u8 *consts = tctx->consts;
 889
 890         if (ctx->len != AES_BLOCK_SIZE) {
 891                 ctx->dg[ctx->len] ^= 0x80;
 892                 consts += AES_BLOCK_SIZE;
 893         }
 894
 895         mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1);
 896
 897         memcpy(out, ctx->dg, AES_BLOCK_SIZE);
 898
 899         return 0;
 900 }
 901
 902 static struct shash_alg mac_algs[] = { {
 903         .base.cra_name          = "cmac(aes)",
 904         .base.cra_driver_name   = "cmac-aes-" MODE,
 905         .base.cra_priority      = PRIO,
 906         .base.cra_blocksize     = AES_BLOCK_SIZE,
 907         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx) +
 908                                   2 * AES_BLOCK_SIZE,
 909         .base.cra_module        = THIS_MODULE,
 910
 911         .digestsize             = AES_BLOCK_SIZE,
 912         .init                   = mac_init,
 913         .update                 = mac_update,
 914         .final                  = cmac_final,
 915         .setkey                 = cmac_setkey,
 916         .descsize               = sizeof(struct mac_desc_ctx),
 917 }, {
 918         .base.cra_name          = "xcbc(aes)",
 919         .base.cra_driver_name   = "xcbc-aes-" MODE,
 920         .base.cra_priority      = PRIO,
 921         .base.cra_blocksize     = AES_BLOCK_SIZE,
 922         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx) +
 923                                   2 * AES_BLOCK_SIZE,
 924         .base.cra_module        = THIS_MODULE,
 925
 926         .digestsize             = AES_BLOCK_SIZE,
 927         .init                   = mac_init,
 928         .update                 = mac_update,
 929         .final                  = cmac_final,
 930         .setkey                 = xcbc_setkey,
 931         .descsize               = sizeof(struct mac_desc_ctx),
 932 }, {
 933         .base.cra_name          = "cbcmac(aes)",
 934         .base.cra_driver_name   = "cbcmac-aes-" MODE,
 935         .base.cra_priority      = PRIO,
 936         .base.cra_blocksize     = 1,
 937         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx),
 938         .base.cra_module        = THIS_MODULE,
 939
 940         .digestsize             = AES_BLOCK_SIZE,
 941         .init                   = mac_init,
 942         .update                 = mac_update,
 943         .final                  = cbcmac_final,
 944         .setkey                 = cbcmac_setkey,
 945         .descsize               = sizeof(struct mac_desc_ctx),
 946 } };
 947
 948 static void aes_exit(void)
 949 {
 950         crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
 951         crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 952 }
 953
 954 static int __init aes_init(void)
 955 {
 956         int err;
 957
 958         err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 959         if (err)
 960                 return err;
 961
 962         err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
 963         if (err)
 964                 goto unregister_ciphers;
 965
 966         return 0;
 967
 968 unregister_ciphers:
 969         crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 970         return err;
 971 }
 972
 973 #ifdef USE_V8_CRYPTO_EXTENSIONS
 974 module_cpu_feature_match(AES, aes_init);
 975 #else
 976 module_init(aes_init);
 977 EXPORT_SYMBOL(neon_aes_ecb_encrypt);
 978 EXPORT_SYMBOL(neon_aes_cbc_encrypt);
 979 EXPORT_SYMBOL(neon_aes_ctr_encrypt);
 980 EXPORT_SYMBOL(neon_aes_xts_encrypt);
 981 EXPORT_SYMBOL(neon_aes_xts_decrypt);
 982 #endif
 983 module_exit(aes_exit);