arch/x86/crypto/twofish_avx_glue.c

   1 /*
   2  * Glue Code for AVX assembler version of Twofish Cipher
   3  *
   4  * Copyright (C) 2012 Johannes Goetzfried
   5  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  20  * USA
  21  *
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/hardirq.h>
  26 #include <linux/types.h>
  27 #include <linux/crypto.h>
  28 #include <linux/err.h>
  29 #include <crypto/algapi.h>
  30 #include <crypto/twofish.h>
  31 #include <crypto/cryptd.h>
  32 #include <crypto/b128ops.h>
  33 #include <crypto/ctr.h>
  34 #include <crypto/lrw.h>
  35 #include <crypto/xts.h>
  36 #include <asm/i387.h>
  37 #include <asm/xcr.h>
  38 #include <asm/xsave.h>
  39 #include <asm/crypto/twofish.h>
  40 #include <asm/crypto/ablk_helper.h>
  41 #include <asm/crypto/glue_helper.h>
  42 #include <crypto/scatterwalk.h>
  43 #include <linux/workqueue.h>
  44 #include <linux/spinlock.h>
  45
  46 #define TWOFISH_PARALLEL_BLOCKS 8
  47
  48 /* 8-way parallel cipher functions */
  49 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
  50                                      const u8 *src);
  51 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  52                                      const u8 *src);
  53
  54 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  55                                      const u8 *src);
  56 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
  57                                  const u8 *src, le128 *iv);
  58
  59 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  60                                         const u8 *src)
  61 {
  62         __twofish_enc_blk_3way(ctx, dst, src, false);
  63 }
  64
  65
  66 static const struct common_glue_ctx twofish_enc = {
  67         .num_funcs = 3,
  68         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
  69
  70         .funcs = { {
  71                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
  72                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
  73         }, {
  74                 .num_blocks = 3,
  75                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
  76         }, {
  77                 .num_blocks = 1,
  78                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
  79         } }
  80 };
  81
  82 static const struct common_glue_ctx twofish_ctr = {
  83         .num_funcs = 3,
  84         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
  85
  86         .funcs = { {
  87                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
  88                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
  89         }, {
  90                 .num_blocks = 3,
  91                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
  92         }, {
  93                 .num_blocks = 1,
  94                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
  95         } }
  96 };
  97
  98 static const struct common_glue_ctx twofish_dec = {
  99         .num_funcs = 3,
 100         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 101
 102         .funcs = { {
 103                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 104                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
 105         }, {
 106                 .num_blocks = 3,
 107                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
 108         }, {
 109                 .num_blocks = 1,
 110                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
 111         } }
 112 };
 113
 114 static const struct common_glue_ctx twofish_dec_cbc = {
 115         .num_funcs = 3,
 116         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 117
 118         .funcs = { {
 119                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 120                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
 121         }, {
 122                 .num_blocks = 3,
 123                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
 124         }, {
 125                 .num_blocks = 1,
 126                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
 127         } }
 128 };
 129
 130 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 131                        struct scatterlist *src, unsigned int nbytes)
 132 {
 133         return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
 134 }
 135
 136 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 137                        struct scatterlist *src, unsigned int nbytes)
 138 {
 139         return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
 140 }
 141
 142 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 143                        struct scatterlist *src, unsigned int nbytes)
 144 {
 145         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
 146                                        dst, src, nbytes);
 147 }
 148
 149 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 150                        struct scatterlist *src, unsigned int nbytes)
 151 {
 152         return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
 153                                        nbytes);
 154 }
 155
 156 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 157                      struct scatterlist *src, unsigned int nbytes)
 158 {
 159         return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
 160 }
 161
 162 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
 163 {
 164         return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
 165                               fpu_enabled, nbytes);
 166 }
 167
 168 static inline void twofish_fpu_end(bool fpu_enabled)
 169 {
 170         glue_fpu_end(fpu_enabled);
 171 }
 172
 173 struct crypt_priv {
 174         struct twofish_ctx *ctx;
 175         bool fpu_enabled;
 176 };
 177
 178 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 179 {
 180         const unsigned int bsize = TF_BLOCK_SIZE;
 181         struct crypt_priv *ctx = priv;
 182         int i;
 183
 184         ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
 185
 186         if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
 187                 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
 188                 return;
 189         }
 190
 191         for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
 192                 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
 193
 194         nbytes %= bsize * 3;
 195
 196         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 197                 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
 198 }
 199
 200 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 201 {
 202         const unsigned int bsize = TF_BLOCK_SIZE;
 203         struct crypt_priv *ctx = priv;
 204         int i;
 205
 206         ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
 207
 208         if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
 209                 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
 210                 return;
 211         }
 212
 213         for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
 214                 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
 215
 216         nbytes %= bsize * 3;
 217
 218         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 219                 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
 220 }
 221
 222 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 223                        struct scatterlist *src, unsigned int nbytes)
 224 {
 225         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 226         be128 buf[TWOFISH_PARALLEL_BLOCKS];
 227         struct crypt_priv crypt_ctx = {
 228                 .ctx = &ctx->twofish_ctx,
 229                 .fpu_enabled = false,
 230         };
 231         struct lrw_crypt_req req = {
 232                 .tbuf = buf,
 233                 .tbuflen = sizeof(buf),
 234
 235                 .table_ctx = &ctx->lrw_table,
 236                 .crypt_ctx = &crypt_ctx,
 237                 .crypt_fn = encrypt_callback,
 238         };
 239         int ret;
 240
 241         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 242         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 243         twofish_fpu_end(crypt_ctx.fpu_enabled);
 244
 245         return ret;
 246 }
 247
 248 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 249                        struct scatterlist *src, unsigned int nbytes)
 250 {
 251         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 252         be128 buf[TWOFISH_PARALLEL_BLOCKS];
 253         struct crypt_priv crypt_ctx = {
 254                 .ctx = &ctx->twofish_ctx,
 255                 .fpu_enabled = false,
 256         };
 257         struct lrw_crypt_req req = {
 258                 .tbuf = buf,
 259                 .tbuflen = sizeof(buf),
 260
 261                 .table_ctx = &ctx->lrw_table,
 262                 .crypt_ctx = &crypt_ctx,
 263                 .crypt_fn = decrypt_callback,
 264         };
 265         int ret;
 266
 267         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 268         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 269         twofish_fpu_end(crypt_ctx.fpu_enabled);
 270
 271         return ret;
 272 }
 273
 274 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 275                        struct scatterlist *src, unsigned int nbytes)
 276 {
 277         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 278         be128 buf[TWOFISH_PARALLEL_BLOCKS];
 279         struct crypt_priv crypt_ctx = {
 280                 .ctx = &ctx->crypt_ctx,
 281                 .fpu_enabled = false,
 282         };
 283         struct xts_crypt_req req = {
 284                 .tbuf = buf,
 285                 .tbuflen = sizeof(buf),
 286
 287                 .tweak_ctx = &ctx->tweak_ctx,
 288                 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
 289                 .crypt_ctx = &crypt_ctx,
 290                 .crypt_fn = encrypt_callback,
 291         };
 292         int ret;
 293
 294         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 295         ret = xts_crypt(desc, dst, src, nbytes, &req);
 296         twofish_fpu_end(crypt_ctx.fpu_enabled);
 297
 298         return ret;
 299 }
 300
 301 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 302                        struct scatterlist *src, unsigned int nbytes)
 303 {
 304         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 305         be128 buf[TWOFISH_PARALLEL_BLOCKS];
 306         struct crypt_priv crypt_ctx = {
 307                 .ctx = &ctx->crypt_ctx,
 308                 .fpu_enabled = false,
 309         };
 310         struct xts_crypt_req req = {
 311                 .tbuf = buf,
 312                 .tbuflen = sizeof(buf),
 313
 314                 .tweak_ctx = &ctx->tweak_ctx,
 315                 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
 316                 .crypt_ctx = &crypt_ctx,
 317                 .crypt_fn = decrypt_callback,
 318         };
 319         int ret;
 320
 321         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 322         ret = xts_crypt(desc, dst, src, nbytes, &req);
 323         twofish_fpu_end(crypt_ctx.fpu_enabled);
 324
 325         return ret;
 326 }
 327
 328 static struct crypto_alg twofish_algs[10] = { {
 329         .cra_name               = "__ecb-twofish-avx",
 330         .cra_driver_name        = "__driver-ecb-twofish-avx",
 331         .cra_priority           = 0,
 332         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 333         .cra_blocksize          = TF_BLOCK_SIZE,
 334         .cra_ctxsize            = sizeof(struct twofish_ctx),
 335         .cra_alignmask          = 0,
 336         .cra_type               = &crypto_blkcipher_type,
 337         .cra_module             = THIS_MODULE,
 338         .cra_u = {
 339                 .blkcipher = {
 340                         .min_keysize    = TF_MIN_KEY_SIZE,
 341                         .max_keysize    = TF_MAX_KEY_SIZE,
 342                         .setkey         = twofish_setkey,
 343                         .encrypt        = ecb_encrypt,
 344                         .decrypt        = ecb_decrypt,
 345                 },
 346         },
 347 }, {
 348         .cra_name               = "__cbc-twofish-avx",
 349         .cra_driver_name        = "__driver-cbc-twofish-avx",
 350         .cra_priority           = 0,
 351         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 352         .cra_blocksize          = TF_BLOCK_SIZE,
 353         .cra_ctxsize            = sizeof(struct twofish_ctx),
 354         .cra_alignmask          = 0,
 355         .cra_type               = &crypto_blkcipher_type,
 356         .cra_module             = THIS_MODULE,
 357         .cra_u = {
 358                 .blkcipher = {
 359                         .min_keysize    = TF_MIN_KEY_SIZE,
 360                         .max_keysize    = TF_MAX_KEY_SIZE,
 361                         .setkey         = twofish_setkey,
 362                         .encrypt        = cbc_encrypt,
 363                         .decrypt        = cbc_decrypt,
 364                 },
 365         },
 366 }, {
 367         .cra_name               = "__ctr-twofish-avx",
 368         .cra_driver_name        = "__driver-ctr-twofish-avx",
 369         .cra_priority           = 0,
 370         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 371         .cra_blocksize          = 1,
 372         .cra_ctxsize            = sizeof(struct twofish_ctx),
 373         .cra_alignmask          = 0,
 374         .cra_type               = &crypto_blkcipher_type,
 375         .cra_module             = THIS_MODULE,
 376         .cra_u = {
 377                 .blkcipher = {
 378                         .min_keysize    = TF_MIN_KEY_SIZE,
 379                         .max_keysize    = TF_MAX_KEY_SIZE,
 380                         .ivsize         = TF_BLOCK_SIZE,
 381                         .setkey         = twofish_setkey,
 382                         .encrypt        = ctr_crypt,
 383                         .decrypt        = ctr_crypt,
 384                 },
 385         },
 386 }, {
 387         .cra_name               = "__lrw-twofish-avx",
 388         .cra_driver_name        = "__driver-lrw-twofish-avx",
 389         .cra_priority           = 0,
 390         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 391         .cra_blocksize          = TF_BLOCK_SIZE,
 392         .cra_ctxsize            = sizeof(struct twofish_lrw_ctx),
 393         .cra_alignmask          = 0,
 394         .cra_type               = &crypto_blkcipher_type,
 395         .cra_module             = THIS_MODULE,
 396         .cra_exit               = lrw_twofish_exit_tfm,
 397         .cra_u = {
 398                 .blkcipher = {
 399                         .min_keysize    = TF_MIN_KEY_SIZE +
 400                                           TF_BLOCK_SIZE,
 401                         .max_keysize    = TF_MAX_KEY_SIZE +
 402                                           TF_BLOCK_SIZE,
 403                         .ivsize         = TF_BLOCK_SIZE,
 404                         .setkey         = lrw_twofish_setkey,
 405                         .encrypt        = lrw_encrypt,
 406                         .decrypt        = lrw_decrypt,
 407                 },
 408         },
 409 }, {
 410         .cra_name               = "__xts-twofish-avx",
 411         .cra_driver_name        = "__driver-xts-twofish-avx",
 412         .cra_priority           = 0,
 413         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 414         .cra_blocksize          = TF_BLOCK_SIZE,
 415         .cra_ctxsize            = sizeof(struct twofish_xts_ctx),
 416         .cra_alignmask          = 0,
 417         .cra_type               = &crypto_blkcipher_type,
 418         .cra_module             = THIS_MODULE,
 419         .cra_u = {
 420                 .blkcipher = {
 421                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
 422                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
 423                         .ivsize         = TF_BLOCK_SIZE,
 424                         .setkey         = xts_twofish_setkey,
 425                         .encrypt        = xts_encrypt,
 426                         .decrypt        = xts_decrypt,
 427                 },
 428         },
 429 }, {
 430         .cra_name               = "ecb(twofish)",
 431         .cra_driver_name        = "ecb-twofish-avx",
 432         .cra_priority           = 400,
 433         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 434         .cra_blocksize          = TF_BLOCK_SIZE,
 435         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 436         .cra_alignmask          = 0,
 437         .cra_type               = &crypto_ablkcipher_type,
 438         .cra_module             = THIS_MODULE,
 439         .cra_init               = ablk_init,
 440         .cra_exit               = ablk_exit,
 441         .cra_u = {
 442                 .ablkcipher = {
 443                         .min_keysize    = TF_MIN_KEY_SIZE,
 444                         .max_keysize    = TF_MAX_KEY_SIZE,
 445                         .setkey         = ablk_set_key,
 446                         .encrypt        = ablk_encrypt,
 447                         .decrypt        = ablk_decrypt,
 448                 },
 449         },
 450 }, {
 451         .cra_name               = "cbc(twofish)",
 452         .cra_driver_name        = "cbc-twofish-avx",
 453         .cra_priority           = 400,
 454         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 455         .cra_blocksize          = TF_BLOCK_SIZE,
 456         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 457         .cra_alignmask          = 0,
 458         .cra_type               = &crypto_ablkcipher_type,
 459         .cra_module             = THIS_MODULE,
 460         .cra_init               = ablk_init,
 461         .cra_exit               = ablk_exit,
 462         .cra_u = {
 463                 .ablkcipher = {
 464                         .min_keysize    = TF_MIN_KEY_SIZE,
 465                         .max_keysize    = TF_MAX_KEY_SIZE,
 466                         .ivsize         = TF_BLOCK_SIZE,
 467                         .setkey         = ablk_set_key,
 468                         .encrypt        = __ablk_encrypt,
 469                         .decrypt        = ablk_decrypt,
 470                 },
 471         },
 472 }, {
 473         .cra_name               = "ctr(twofish)",
 474         .cra_driver_name        = "ctr-twofish-avx",
 475         .cra_priority           = 400,
 476         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 477         .cra_blocksize          = 1,
 478         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 479         .cra_alignmask          = 0,
 480         .cra_type               = &crypto_ablkcipher_type,
 481         .cra_module             = THIS_MODULE,
 482         .cra_init               = ablk_init,
 483         .cra_exit               = ablk_exit,
 484         .cra_u = {
 485                 .ablkcipher = {
 486                         .min_keysize    = TF_MIN_KEY_SIZE,
 487                         .max_keysize    = TF_MAX_KEY_SIZE,
 488                         .ivsize         = TF_BLOCK_SIZE,
 489                         .setkey         = ablk_set_key,
 490                         .encrypt        = ablk_encrypt,
 491                         .decrypt        = ablk_encrypt,
 492                         .geniv          = "chainiv",
 493                 },
 494         },
 495 }, {
 496         .cra_name               = "lrw(twofish)",
 497         .cra_driver_name        = "lrw-twofish-avx",
 498         .cra_priority           = 400,
 499         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 500         .cra_blocksize          = TF_BLOCK_SIZE,
 501         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 502         .cra_alignmask          = 0,
 503         .cra_type               = &crypto_ablkcipher_type,
 504         .cra_module             = THIS_MODULE,
 505         .cra_init               = ablk_init,
 506         .cra_exit               = ablk_exit,
 507         .cra_u = {
 508                 .ablkcipher = {
 509                         .min_keysize    = TF_MIN_KEY_SIZE +
 510                                           TF_BLOCK_SIZE,
 511                         .max_keysize    = TF_MAX_KEY_SIZE +
 512                                           TF_BLOCK_SIZE,
 513                         .ivsize         = TF_BLOCK_SIZE,
 514                         .setkey         = ablk_set_key,
 515                         .encrypt        = ablk_encrypt,
 516                         .decrypt        = ablk_decrypt,
 517                 },
 518         },
 519 }, {
 520         .cra_name               = "xts(twofish)",
 521         .cra_driver_name        = "xts-twofish-avx",
 522         .cra_priority           = 400,
 523         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 524         .cra_blocksize          = TF_BLOCK_SIZE,
 525         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 526         .cra_alignmask          = 0,
 527         .cra_type               = &crypto_ablkcipher_type,
 528         .cra_module             = THIS_MODULE,
 529         .cra_init               = ablk_init,
 530         .cra_exit               = ablk_exit,
 531         .cra_u = {
 532                 .ablkcipher = {
 533                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
 534                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
 535                         .ivsize         = TF_BLOCK_SIZE,
 536                         .setkey         = ablk_set_key,
 537                         .encrypt        = ablk_encrypt,
 538                         .decrypt        = ablk_decrypt,
 539                 },
 540         },
 541 } };
 542
 543 static int __init twofish_init(void)
 544 {
 545         u64 xcr0;
 546
 547         if (!cpu_has_avx || !cpu_has_osxsave) {
 548                 printk(KERN_INFO "AVX instructions are not detected.\n");
 549                 return -ENODEV;
 550         }
 551
 552         xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
 553         if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
 554                 printk(KERN_INFO "AVX detected but unusable.\n");
 555                 return -ENODEV;
 556         }
 557
 558         return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
 559 }
 560
 561 static void __exit twofish_exit(void)
 562 {
 563         crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
 564 }
 565
 566 module_init(twofish_init);
 567 module_exit(twofish_exit);
 568
 569 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
 570 MODULE_LICENSE("GPL");
 571 MODULE_ALIAS("twofish");