crypto/tcrypt.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Quick & dirty crypto testing module.
   4  *
   5  * This will only exist until we have a better testing mechanism
   6  * (e.g. a char device).
   7  *
   8  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
   9  * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
  10  * Copyright (c) 2007 Nokia Siemens Networks
  11  *
  12  * Updated RFC4106 AES-GCM testing.
  13  *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *             Adrian Hoban <adrian.hoban@intel.com>
  15  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  16  *             Tadeusz Struk (tadeusz.struk@intel.com)
  17  *             Copyright (c) 2010, Intel Corporation.
  18  */
  19
  20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  21
  22 #include <crypto/aead.h>
  23 #include <crypto/hash.h>
  24 #include <crypto/skcipher.h>
  25 #include <linux/err.h>
  26 #include <linux/fips.h>
  27 #include <linux/init.h>
  28 #include <linux/gfp.h>
  29 #include <linux/module.h>
  30 #include <linux/scatterlist.h>
  31 #include <linux/string.h>
  32 #include <linux/moduleparam.h>
  33 #include <linux/jiffies.h>
  34 #include <linux/timex.h>
  35 #include <linux/interrupt.h>
  36 #include "tcrypt.h"
  37
  38 /*
  39  * Need slab memory for testing (size in number of pages).
  40  */
  41 #define TVMEMSIZE       4
  42
  43 /*
  44 * Used by test_cipher_speed()
  45 */
  46 #define ENCRYPT 1
  47 #define DECRYPT 0
  48
  49 #define MAX_DIGEST_SIZE         64
  50
  51 /*
  52  * return a string with the driver name
  53  */
  54 #define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm))
  55
  56 /*
  57  * Used by test_cipher_speed()
  58  */
  59 static unsigned int sec;
  60
  61 static char *alg;
  62 static u32 type;
  63 static u32 mask;
  64 static int mode;
  65 static u32 num_mb = 8;
  66 static unsigned int klen;
  67 static char *tvmem[TVMEMSIZE];
  68
  69 static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 };
  70 static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 };
  71
  72 #define XBUFSIZE 8
  73 #define MAX_IVLEN 32
  74
  75 static int testmgr_alloc_buf(char *buf[XBUFSIZE])
  76 {
  77         int i;
  78
  79         for (i = 0; i < XBUFSIZE; i++) {
  80                 buf[i] = (void *)__get_free_page(GFP_KERNEL);
  81                 if (!buf[i])
  82                         goto err_free_buf;
  83         }
  84
  85         return 0;
  86
  87 err_free_buf:
  88         while (i-- > 0)
  89                 free_page((unsigned long)buf[i]);
  90
  91         return -ENOMEM;
  92 }
  93
  94 static void testmgr_free_buf(char *buf[XBUFSIZE])
  95 {
  96         int i;
  97
  98         for (i = 0; i < XBUFSIZE; i++)
  99                 free_page((unsigned long)buf[i]);
 100 }
 101
 102 static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
 103                          unsigned int buflen, const void *assoc,
 104                          unsigned int aad_size)
 105 {
 106         int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
 107         int k, rem;
 108
 109         if (np > XBUFSIZE) {
 110                 rem = PAGE_SIZE;
 111                 np = XBUFSIZE;
 112         } else {
 113                 rem = buflen % PAGE_SIZE;
 114         }
 115
 116         sg_init_table(sg, np + 1);
 117
 118         sg_set_buf(&sg[0], assoc, aad_size);
 119
 120         if (rem)
 121                 np--;
 122         for (k = 0; k < np; k++)
 123                 sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
 124
 125         if (rem)
 126                 sg_set_buf(&sg[k + 1], xbuf[k], rem);
 127 }
 128
 129 static inline int do_one_aead_op(struct aead_request *req, int ret)
 130 {
 131         struct crypto_wait *wait = req->base.data;
 132
 133         return crypto_wait_req(ret, wait);
 134 }
 135
 136 struct test_mb_aead_data {
 137         struct scatterlist sg[XBUFSIZE];
 138         struct scatterlist sgout[XBUFSIZE];
 139         struct aead_request *req;
 140         struct crypto_wait wait;
 141         char *xbuf[XBUFSIZE];
 142         char *xoutbuf[XBUFSIZE];
 143         char *axbuf[XBUFSIZE];
 144 };
 145
 146 static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
 147                                 u32 num_mb, int *rc)
 148 {
 149         int i, err = 0;
 150
 151         /* Fire up a bunch of concurrent requests */
 152         for (i = 0; i < num_mb; i++) {
 153                 if (enc == ENCRYPT)
 154                         rc[i] = crypto_aead_encrypt(data[i].req);
 155                 else
 156                         rc[i] = crypto_aead_decrypt(data[i].req);
 157         }
 158
 159         /* Wait for all requests to finish */
 160         for (i = 0; i < num_mb; i++) {
 161                 rc[i] = crypto_wait_req(rc[i], &data[i].wait);
 162
 163                 if (rc[i]) {
 164                         pr_info("concurrent request %d error %d\n", i, rc[i]);
 165                         err = rc[i];
 166                 }
 167         }
 168
 169         return err;
 170 }
 171
 172 static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
 173                                 int blen, int secs, u32 num_mb)
 174 {
 175         unsigned long start, end;
 176         int bcount;
 177         int ret = 0;
 178         int *rc;
 179
 180         rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
 181         if (!rc)
 182                 return -ENOMEM;
 183
 184         for (start = jiffies, end = start + secs * HZ, bcount = 0;
 185              time_before(jiffies, end); bcount++) {
 186                 ret = do_mult_aead_op(data, enc, num_mb, rc);
 187                 if (ret)
 188                         goto out;
 189         }
 190
 191         pr_cont("%d operations in %d seconds (%llu bytes)\n",
 192                 bcount * num_mb, secs, (u64)bcount * blen * num_mb);
 193
 194 out:
 195         kfree(rc);
 196         return ret;
 197 }
 198
 199 static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
 200                                int blen, u32 num_mb)
 201 {
 202         unsigned long cycles = 0;
 203         int ret = 0;
 204         int i;
 205         int *rc;
 206
 207         rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
 208         if (!rc)
 209                 return -ENOMEM;
 210
 211         /* Warm-up run. */
 212         for (i = 0; i < 4; i++) {
 213                 ret = do_mult_aead_op(data, enc, num_mb, rc);
 214                 if (ret)
 215                         goto out;
 216         }
 217
 218         /* The real thing. */
 219         for (i = 0; i < 8; i++) {
 220                 cycles_t start, end;
 221
 222                 start = get_cycles();
 223                 ret = do_mult_aead_op(data, enc, num_mb, rc);
 224                 end = get_cycles();
 225
 226                 if (ret)
 227                         goto out;
 228
 229                 cycles += end - start;
 230         }
 231
 232         pr_cont("1 operation in %lu cycles (%d bytes)\n",
 233                 (cycles + 4) / (8 * num_mb), blen);
 234
 235 out:
 236         kfree(rc);
 237         return ret;
 238 }
 239
 240 static void test_mb_aead_speed(const char *algo, int enc, int secs,
 241                                struct aead_speed_template *template,
 242                                unsigned int tcount, u8 authsize,
 243                                unsigned int aad_size, u8 *keysize, u32 num_mb)
 244 {
 245         struct test_mb_aead_data *data;
 246         struct crypto_aead *tfm;
 247         unsigned int i, j, iv_len;
 248         const int *b_size;
 249         const char *key;
 250         const char *e;
 251         void *assoc;
 252         char *iv;
 253         int ret;
 254
 255
 256         if (aad_size >= PAGE_SIZE) {
 257                 pr_err("associate data length (%u) too big\n", aad_size);
 258                 return;
 259         }
 260
 261         iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
 262         if (!iv)
 263                 return;
 264
 265         if (enc == ENCRYPT)
 266                 e = "encryption";
 267         else
 268                 e = "decryption";
 269
 270         data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
 271         if (!data)
 272                 goto out_free_iv;
 273
 274         tfm = crypto_alloc_aead(algo, 0, 0);
 275         if (IS_ERR(tfm)) {
 276                 pr_err("failed to load transform for %s: %ld\n",
 277                         algo, PTR_ERR(tfm));
 278                 goto out_free_data;
 279         }
 280
 281         ret = crypto_aead_setauthsize(tfm, authsize);
 282         if (ret) {
 283                 pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
 284                        ret);
 285                 goto out_free_tfm;
 286         }
 287
 288         for (i = 0; i < num_mb; ++i)
 289                 if (testmgr_alloc_buf(data[i].xbuf)) {
 290                         while (i--)
 291                                 testmgr_free_buf(data[i].xbuf);
 292                         goto out_free_tfm;
 293                 }
 294
 295         for (i = 0; i < num_mb; ++i)
 296                 if (testmgr_alloc_buf(data[i].axbuf)) {
 297                         while (i--)
 298                                 testmgr_free_buf(data[i].axbuf);
 299                         goto out_free_xbuf;
 300                 }
 301
 302         for (i = 0; i < num_mb; ++i)
 303                 if (testmgr_alloc_buf(data[i].xoutbuf)) {
 304                         while (i--)
 305                                 testmgr_free_buf(data[i].xoutbuf);
 306                         goto out_free_axbuf;
 307                 }
 308
 309         for (i = 0; i < num_mb; ++i) {
 310                 data[i].req = aead_request_alloc(tfm, GFP_KERNEL);
 311                 if (!data[i].req) {
 312                         pr_err("alg: aead: Failed to allocate request for %s\n",
 313                                algo);
 314                         while (i--)
 315                                 aead_request_free(data[i].req);
 316                         goto out_free_xoutbuf;
 317                 }
 318         }
 319
 320         for (i = 0; i < num_mb; ++i) {
 321                 crypto_init_wait(&data[i].wait);
 322                 aead_request_set_callback(data[i].req,
 323                                           CRYPTO_TFM_REQ_MAY_BACKLOG,
 324                                           crypto_req_done, &data[i].wait);
 325         }
 326
 327         pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
 328                 get_driver_name(crypto_aead, tfm), e);
 329
 330         i = 0;
 331         do {
 332                 b_size = aead_sizes;
 333                 do {
 334                         int bs = round_up(*b_size, crypto_aead_blocksize(tfm));
 335
 336                         if (bs + authsize > XBUFSIZE * PAGE_SIZE) {
 337                                 pr_err("template (%u) too big for buffer (%lu)\n",
 338                                        authsize + bs,
 339                                        XBUFSIZE * PAGE_SIZE);
 340                                 goto out;
 341                         }
 342
 343                         pr_info("test %u (%d bit key, %d byte blocks): ", i,
 344                                 *keysize * 8, bs);
 345
 346                         /* Set up tfm global state, i.e. the key */
 347
 348                         memset(tvmem[0], 0xff, PAGE_SIZE);
 349                         key = tvmem[0];
 350                         for (j = 0; j < tcount; j++) {
 351                                 if (template[j].klen == *keysize) {
 352                                         key = template[j].key;
 353                                         break;
 354                                 }
 355                         }
 356
 357                         crypto_aead_clear_flags(tfm, ~0);
 358
 359                         ret = crypto_aead_setkey(tfm, key, *keysize);
 360                         if (ret) {
 361                                 pr_err("setkey() failed flags=%x\n",
 362                                        crypto_aead_get_flags(tfm));
 363                                 goto out;
 364                         }
 365
 366                         iv_len = crypto_aead_ivsize(tfm);
 367                         if (iv_len)
 368                                 memset(iv, 0xff, iv_len);
 369
 370                         /* Now setup per request stuff, i.e. buffers */
 371
 372                         for (j = 0; j < num_mb; ++j) {
 373                                 struct test_mb_aead_data *cur = &data[j];
 374
 375                                 assoc = cur->axbuf[0];
 376                                 memset(assoc, 0xff, aad_size);
 377
 378                                 sg_init_aead(cur->sg, cur->xbuf,
 379                                              bs + (enc ? 0 : authsize),
 380                                              assoc, aad_size);
 381
 382                                 sg_init_aead(cur->sgout, cur->xoutbuf,
 383                                              bs + (enc ? authsize : 0),
 384                                              assoc, aad_size);
 385
 386                                 aead_request_set_ad(cur->req, aad_size);
 387
 388                                 if (!enc) {
 389
 390                                         aead_request_set_crypt(cur->req,
 391                                                                cur->sgout,
 392                                                                cur->sg,
 393                                                                bs, iv);
 394                                         ret = crypto_aead_encrypt(cur->req);
 395                                         ret = do_one_aead_op(cur->req, ret);
 396
 397                                         if (ret) {
 398                                                 pr_err("calculating auth failed (%d)\n",
 399                                                        ret);
 400                                                 break;
 401                                         }
 402                                 }
 403
 404                                 aead_request_set_crypt(cur->req, cur->sg,
 405                                                        cur->sgout, bs +
 406                                                        (enc ? 0 : authsize),
 407                                                        iv);
 408
 409                         }
 410
 411                         if (secs) {
 412                                 ret = test_mb_aead_jiffies(data, enc, bs,
 413                                                            secs, num_mb);
 414                                 cond_resched();
 415                         } else {
 416                                 ret = test_mb_aead_cycles(data, enc, bs,
 417                                                           num_mb);
 418                         }
 419
 420                         if (ret) {
 421                                 pr_err("%s() failed return code=%d\n", e, ret);
 422                                 break;
 423                         }
 424                         b_size++;
 425                         i++;
 426                 } while (*b_size);
 427                 keysize++;
 428         } while (*keysize);
 429
 430 out:
 431         for (i = 0; i < num_mb; ++i)
 432                 aead_request_free(data[i].req);
 433 out_free_xoutbuf:
 434         for (i = 0; i < num_mb; ++i)
 435                 testmgr_free_buf(data[i].xoutbuf);
 436 out_free_axbuf:
 437         for (i = 0; i < num_mb; ++i)
 438                 testmgr_free_buf(data[i].axbuf);
 439 out_free_xbuf:
 440         for (i = 0; i < num_mb; ++i)
 441                 testmgr_free_buf(data[i].xbuf);
 442 out_free_tfm:
 443         crypto_free_aead(tfm);
 444 out_free_data:
 445         kfree(data);
 446 out_free_iv:
 447         kfree(iv);
 448 }
 449
 450 static int test_aead_jiffies(struct aead_request *req, int enc,
 451                                 int blen, int secs)
 452 {
 453         unsigned long start, end;
 454         int bcount;
 455         int ret;
 456
 457         for (start = jiffies, end = start + secs * HZ, bcount = 0;
 458              time_before(jiffies, end); bcount++) {
 459                 if (enc)
 460                         ret = do_one_aead_op(req, crypto_aead_encrypt(req));
 461                 else
 462                         ret = do_one_aead_op(req, crypto_aead_decrypt(req));
 463
 464                 if (ret)
 465                         return ret;
 466         }
 467
 468         pr_cont("%d operations in %d seconds (%llu bytes)\n",
 469                 bcount, secs, (u64)bcount * blen);
 470         return 0;
 471 }
 472
 473 static int test_aead_cycles(struct aead_request *req, int enc, int blen)
 474 {
 475         unsigned long cycles = 0;
 476         int ret = 0;
 477         int i;
 478
 479         /* Warm-up run. */
 480         for (i = 0; i < 4; i++) {
 481                 if (enc)
 482                         ret = do_one_aead_op(req, crypto_aead_encrypt(req));
 483                 else
 484                         ret = do_one_aead_op(req, crypto_aead_decrypt(req));
 485
 486                 if (ret)
 487                         goto out;
 488         }
 489
 490         /* The real thing. */
 491         for (i = 0; i < 8; i++) {
 492                 cycles_t start, end;
 493
 494                 start = get_cycles();
 495                 if (enc)
 496                         ret = do_one_aead_op(req, crypto_aead_encrypt(req));
 497                 else
 498                         ret = do_one_aead_op(req, crypto_aead_decrypt(req));
 499                 end = get_cycles();
 500
 501                 if (ret)
 502                         goto out;
 503
 504                 cycles += end - start;
 505         }
 506
 507 out:
 508         if (ret == 0)
 509                 printk("1 operation in %lu cycles (%d bytes)\n",
 510                        (cycles + 4) / 8, blen);
 511
 512         return ret;
 513 }
 514
 515 static void test_aead_speed(const char *algo, int enc, unsigned int secs,
 516                             struct aead_speed_template *template,
 517                             unsigned int tcount, u8 authsize,
 518                             unsigned int aad_size, u8 *keysize)
 519 {
 520         unsigned int i, j;
 521         struct crypto_aead *tfm;
 522         int ret = -ENOMEM;
 523         const char *key;
 524         struct aead_request *req;
 525         struct scatterlist *sg;
 526         struct scatterlist *sgout;
 527         const char *e;
 528         void *assoc;
 529         char *iv;
 530         char *xbuf[XBUFSIZE];
 531         char *xoutbuf[XBUFSIZE];
 532         char *axbuf[XBUFSIZE];
 533         const int *b_size;
 534         unsigned int iv_len;
 535         struct crypto_wait wait;
 536
 537         iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
 538         if (!iv)
 539                 return;
 540
 541         if (aad_size >= PAGE_SIZE) {
 542                 pr_err("associate data length (%u) too big\n", aad_size);
 543                 goto out_noxbuf;
 544         }
 545
 546         if (enc == ENCRYPT)
 547                 e = "encryption";
 548         else
 549                 e = "decryption";
 550
 551         if (testmgr_alloc_buf(xbuf))
 552                 goto out_noxbuf;
 553         if (testmgr_alloc_buf(axbuf))
 554                 goto out_noaxbuf;
 555         if (testmgr_alloc_buf(xoutbuf))
 556                 goto out_nooutbuf;
 557
 558         sg = kmalloc(sizeof(*sg) * 9 * 2, GFP_KERNEL);
 559         if (!sg)
 560                 goto out_nosg;
 561         sgout = &sg[9];
 562
 563         tfm = crypto_alloc_aead(algo, 0, 0);
 564         if (IS_ERR(tfm)) {
 565                 pr_err("alg: aead: Failed to load transform for %s: %ld\n", algo,
 566                        PTR_ERR(tfm));
 567                 goto out_notfm;
 568         }
 569
 570         ret = crypto_aead_setauthsize(tfm, authsize);
 571         if (ret) {
 572                 pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
 573                        ret);
 574                 goto out_noreq;
 575         }
 576
 577         crypto_init_wait(&wait);
 578         printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo,
 579                         get_driver_name(crypto_aead, tfm), e);
 580
 581         req = aead_request_alloc(tfm, GFP_KERNEL);
 582         if (!req) {
 583                 pr_err("alg: aead: Failed to allocate request for %s\n",
 584                        algo);
 585                 goto out_noreq;
 586         }
 587
 588         aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 589                                   crypto_req_done, &wait);
 590
 591         i = 0;
 592         do {
 593                 b_size = aead_sizes;
 594                 do {
 595                         u32 bs = round_up(*b_size, crypto_aead_blocksize(tfm));
 596
 597                         assoc = axbuf[0];
 598                         memset(assoc, 0xff, aad_size);
 599
 600                         if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
 601                                 pr_err("template (%u) too big for tvmem (%lu)\n",
 602                                        *keysize + bs,
 603                                         TVMEMSIZE * PAGE_SIZE);
 604                                 goto out;
 605                         }
 606
 607                         key = tvmem[0];
 608                         for (j = 0; j < tcount; j++) {
 609                                 if (template[j].klen == *keysize) {
 610                                         key = template[j].key;
 611                                         break;
 612                                 }
 613                         }
 614
 615                         ret = crypto_aead_setkey(tfm, key, *keysize);
 616                         if (ret) {
 617                                 pr_err("setkey() failed flags=%x: %d\n",
 618                                         crypto_aead_get_flags(tfm), ret);
 619                                 goto out;
 620                         }
 621
 622                         iv_len = crypto_aead_ivsize(tfm);
 623                         if (iv_len)
 624                                 memset(iv, 0xff, iv_len);
 625
 626                         crypto_aead_clear_flags(tfm, ~0);
 627                         printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ",
 628                                         i, *keysize * 8, bs);
 629
 630                         memset(tvmem[0], 0xff, PAGE_SIZE);
 631
 632                         sg_init_aead(sg, xbuf, bs + (enc ? 0 : authsize),
 633                                      assoc, aad_size);
 634
 635                         sg_init_aead(sgout, xoutbuf,
 636                                      bs + (enc ? authsize : 0), assoc,
 637                                      aad_size);
 638
 639                         aead_request_set_ad(req, aad_size);
 640
 641                         if (!enc) {
 642
 643                                 /*
 644                                  * For decryption we need a proper auth so
 645                                  * we do the encryption path once with buffers
 646                                  * reversed (input <-> output) to calculate it
 647                                  */
 648                                 aead_request_set_crypt(req, sgout, sg,
 649                                                        bs, iv);
 650                                 ret = do_one_aead_op(req,
 651                                                      crypto_aead_encrypt(req));
 652
 653                                 if (ret) {
 654                                         pr_err("calculating auth failed (%d)\n",
 655                                                ret);
 656                                         break;
 657                                 }
 658                         }
 659
 660                         aead_request_set_crypt(req, sg, sgout,
 661                                                bs + (enc ? 0 : authsize),
 662                                                iv);
 663
 664                         if (secs) {
 665                                 ret = test_aead_jiffies(req, enc, bs,
 666                                                         secs);
 667                                 cond_resched();
 668                         } else {
 669                                 ret = test_aead_cycles(req, enc, bs);
 670                         }
 671
 672                         if (ret) {
 673                                 pr_err("%s() failed return code=%d\n", e, ret);
 674                                 break;
 675                         }
 676                         b_size++;
 677                         i++;
 678                 } while (*b_size);
 679                 keysize++;
 680         } while (*keysize);
 681
 682 out:
 683         aead_request_free(req);
 684 out_noreq:
 685         crypto_free_aead(tfm);
 686 out_notfm:
 687         kfree(sg);
 688 out_nosg:
 689         testmgr_free_buf(xoutbuf);
 690 out_nooutbuf:
 691         testmgr_free_buf(axbuf);
 692 out_noaxbuf:
 693         testmgr_free_buf(xbuf);
 694 out_noxbuf:
 695         kfree(iv);
 696 }
 697
 698 static void test_hash_sg_init(struct scatterlist *sg)
 699 {
 700         int i;
 701
 702         sg_init_table(sg, TVMEMSIZE);
 703         for (i = 0; i < TVMEMSIZE; i++) {
 704                 sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
 705                 memset(tvmem[i], 0xff, PAGE_SIZE);
 706         }
 707 }
 708
 709 static inline int do_one_ahash_op(struct ahash_request *req, int ret)
 710 {
 711         struct crypto_wait *wait = req->base.data;
 712
 713         return crypto_wait_req(ret, wait);
 714 }
 715
 716 static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
 717                                      char *out, int secs)
 718 {
 719         unsigned long start, end;
 720         int bcount;
 721         int ret;
 722
 723         for (start = jiffies, end = start + secs * HZ, bcount = 0;
 724              time_before(jiffies, end); bcount++) {
 725                 ret = do_one_ahash_op(req, crypto_ahash_digest(req));
 726                 if (ret)
 727                         return ret;
 728         }
 729
 730         printk("%6u opers/sec, %9lu bytes/sec\n",
 731                bcount / secs, ((long)bcount * blen) / secs);
 732
 733         return 0;
 734 }
 735
 736 static int test_ahash_jiffies(struct ahash_request *req, int blen,
 737                               int plen, char *out, int secs)
 738 {
 739         unsigned long start, end;
 740         int bcount, pcount;
 741         int ret;
 742
 743         if (plen == blen)
 744                 return test_ahash_jiffies_digest(req, blen, out, secs);
 745
 746         for (start = jiffies, end = start + secs * HZ, bcount = 0;
 747              time_before(jiffies, end); bcount++) {
 748                 ret = do_one_ahash_op(req, crypto_ahash_init(req));
 749                 if (ret)
 750                         return ret;
 751                 for (pcount = 0; pcount < blen; pcount += plen) {
 752                         ret = do_one_ahash_op(req, crypto_ahash_update(req));
 753                         if (ret)
 754                                 return ret;
 755                 }
 756                 /* we assume there is enough space in 'out' for the result */
 757                 ret = do_one_ahash_op(req, crypto_ahash_final(req));
 758                 if (ret)
 759                         return ret;
 760         }
 761
 762         pr_cont("%6u opers/sec, %9lu bytes/sec\n",
 763                 bcount / secs, ((long)bcount * blen) / secs);
 764
 765         return 0;
 766 }
 767
 768 static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
 769                                     char *out)
 770 {
 771         unsigned long cycles = 0;
 772         int ret, i;
 773
 774         /* Warm-up run. */
 775         for (i = 0; i < 4; i++) {
 776                 ret = do_one_ahash_op(req, crypto_ahash_digest(req));
 777                 if (ret)
 778                         goto out;
 779         }
 780
 781         /* The real thing. */
 782         for (i = 0; i < 8; i++) {
 783                 cycles_t start, end;
 784
 785                 start = get_cycles();
 786
 787                 ret = do_one_ahash_op(req, crypto_ahash_digest(req));
 788                 if (ret)
 789                         goto out;
 790
 791                 end = get_cycles();
 792
 793                 cycles += end - start;
 794         }
 795
 796 out:
 797         if (ret)
 798                 return ret;
 799
 800         pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
 801                 cycles / 8, cycles / (8 * blen));
 802
 803         return 0;
 804 }
 805
 806 static int test_ahash_cycles(struct ahash_request *req, int blen,
 807                              int plen, char *out)
 808 {
 809         unsigned long cycles = 0;
 810         int i, pcount, ret;
 811
 812         if (plen == blen)
 813                 return test_ahash_cycles_digest(req, blen, out);
 814
 815         /* Warm-up run. */
 816         for (i = 0; i < 4; i++) {
 817                 ret = do_one_ahash_op(req, crypto_ahash_init(req));
 818                 if (ret)
 819                         goto out;
 820                 for (pcount = 0; pcount < blen; pcount += plen) {
 821                         ret = do_one_ahash_op(req, crypto_ahash_update(req));
 822                         if (ret)
 823                                 goto out;
 824                 }
 825                 ret = do_one_ahash_op(req, crypto_ahash_final(req));
 826                 if (ret)
 827                         goto out;
 828         }
 829
 830         /* The real thing. */
 831         for (i = 0; i < 8; i++) {
 832                 cycles_t start, end;
 833
 834                 start = get_cycles();
 835
 836                 ret = do_one_ahash_op(req, crypto_ahash_init(req));
 837                 if (ret)
 838                         goto out;
 839                 for (pcount = 0; pcount < blen; pcount += plen) {
 840                         ret = do_one_ahash_op(req, crypto_ahash_update(req));
 841                         if (ret)
 842                                 goto out;
 843                 }
 844                 ret = do_one_ahash_op(req, crypto_ahash_final(req));
 845                 if (ret)
 846                         goto out;
 847
 848                 end = get_cycles();
 849
 850                 cycles += end - start;
 851         }
 852
 853 out:
 854         if (ret)
 855                 return ret;
 856
 857         pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
 858                 cycles / 8, cycles / (8 * blen));
 859
 860         return 0;
 861 }
 862
 863 static void test_ahash_speed_common(const char *algo, unsigned int secs,
 864                                     struct hash_speed *speed, unsigned mask)
 865 {
 866         struct scatterlist sg[TVMEMSIZE];
 867         struct crypto_wait wait;
 868         struct ahash_request *req;
 869         struct crypto_ahash *tfm;
 870         char *output;
 871         int i, ret;
 872
 873         tfm = crypto_alloc_ahash(algo, 0, mask);
 874         if (IS_ERR(tfm)) {
 875                 pr_err("failed to load transform for %s: %ld\n",
 876                        algo, PTR_ERR(tfm));
 877                 return;
 878         }
 879
 880         printk(KERN_INFO "\ntesting speed of async %s (%s)\n", algo,
 881                         get_driver_name(crypto_ahash, tfm));
 882
 883         if (crypto_ahash_digestsize(tfm) > MAX_DIGEST_SIZE) {
 884                 pr_err("digestsize(%u) > %d\n", crypto_ahash_digestsize(tfm),
 885                        MAX_DIGEST_SIZE);
 886                 goto out;
 887         }
 888
 889         test_hash_sg_init(sg);
 890         req = ahash_request_alloc(tfm, GFP_KERNEL);
 891         if (!req) {
 892                 pr_err("ahash request allocation failure\n");
 893                 goto out;
 894         }
 895
 896         crypto_init_wait(&wait);
 897         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 898                                    crypto_req_done, &wait);
 899
 900         output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL);
 901         if (!output)
 902                 goto out_nomem;
 903
 904         for (i = 0; speed[i].blen != 0; i++) {
 905                 if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
 906                         pr_err("template (%u) too big for tvmem (%lu)\n",
 907                                speed[i].blen, TVMEMSIZE * PAGE_SIZE);
 908                         break;
 909                 }
 910
 911                 if (klen)
 912                         crypto_ahash_setkey(tfm, tvmem[0], klen);
 913
 914                 pr_info("test%3u "
 915                         "(%5u byte blocks,%5u bytes per update,%4u updates): ",
 916                         i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
 917
 918                 ahash_request_set_crypt(req, sg, output, speed[i].plen);
 919
 920                 if (secs) {
 921                         ret = test_ahash_jiffies(req, speed[i].blen,
 922                                                  speed[i].plen, output, secs);
 923                         cond_resched();
 924                 } else {
 925                         ret = test_ahash_cycles(req, speed[i].blen,
 926                                                 speed[i].plen, output);
 927                 }
 928
 929                 if (ret) {
 930                         pr_err("hashing failed ret=%d\n", ret);
 931                         break;
 932                 }
 933         }
 934
 935         kfree(output);
 936
 937 out_nomem:
 938         ahash_request_free(req);
 939
 940 out:
 941         crypto_free_ahash(tfm);
 942 }
 943
 944 static void test_ahash_speed(const char *algo, unsigned int secs,
 945                              struct hash_speed *speed)
 946 {
 947         return test_ahash_speed_common(algo, secs, speed, 0);
 948 }
 949
 950 static void test_hash_speed(const char *algo, unsigned int secs,
 951                             struct hash_speed *speed)
 952 {
 953         return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
 954 }
 955
 956 struct test_mb_skcipher_data {
 957         struct scatterlist sg[XBUFSIZE];
 958         struct skcipher_request *req;
 959         struct crypto_wait wait;
 960         char *xbuf[XBUFSIZE];
 961 };
 962
 963 static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
 964                                 u32 num_mb, int *rc)
 965 {
 966         int i, err = 0;
 967
 968         /* Fire up a bunch of concurrent requests */
 969         for (i = 0; i < num_mb; i++) {
 970                 if (enc == ENCRYPT)
 971                         rc[i] = crypto_skcipher_encrypt(data[i].req);
 972                 else
 973                         rc[i] = crypto_skcipher_decrypt(data[i].req);
 974         }
 975
 976         /* Wait for all requests to finish */
 977         for (i = 0; i < num_mb; i++) {
 978                 rc[i] = crypto_wait_req(rc[i], &data[i].wait);
 979
 980                 if (rc[i]) {
 981                         pr_info("concurrent request %d error %d\n", i, rc[i]);
 982                         err = rc[i];
 983                 }
 984         }
 985
 986         return err;
 987 }
 988
 989 static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
 990                                 int blen, int secs, u32 num_mb)
 991 {
 992         unsigned long start, end;
 993         int bcount;
 994         int ret = 0;
 995         int *rc;
 996
 997         rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
 998         if (!rc)
 999                 return -ENOMEM;
1000
1001         for (start = jiffies, end = start + secs * HZ, bcount = 0;
1002              time_before(jiffies, end); bcount++) {
1003                 ret = do_mult_acipher_op(data, enc, num_mb, rc);
1004                 if (ret)
1005                         goto out;
1006         }
1007
1008         pr_cont("%d operations in %d seconds (%llu bytes)\n",
1009                 bcount * num_mb, secs, (u64)bcount * blen * num_mb);
1010
1011 out:
1012         kfree(rc);
1013         return ret;
1014 }
1015
1016 static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
1017                                int blen, u32 num_mb)
1018 {
1019         unsigned long cycles = 0;
1020         int ret = 0;
1021         int i;
1022         int *rc;
1023
1024         rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
1025         if (!rc)
1026                 return -ENOMEM;
1027
1028         /* Warm-up run. */
1029         for (i = 0; i < 4; i++) {
1030                 ret = do_mult_acipher_op(data, enc, num_mb, rc);
1031                 if (ret)
1032                         goto out;
1033         }
1034
1035         /* The real thing. */
1036         for (i = 0; i < 8; i++) {
1037                 cycles_t start, end;
1038
1039                 start = get_cycles();
1040                 ret = do_mult_acipher_op(data, enc, num_mb, rc);
1041                 end = get_cycles();
1042
1043                 if (ret)
1044                         goto out;
1045
1046                 cycles += end - start;
1047         }
1048
1049         pr_cont("1 operation in %lu cycles (%d bytes)\n",
1050                 (cycles + 4) / (8 * num_mb), blen);
1051
1052 out:
1053         kfree(rc);
1054         return ret;
1055 }
1056
1057 static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
1058                                    struct cipher_speed_template *template,
1059                                    unsigned int tcount, u8 *keysize, u32 num_mb)
1060 {
1061         struct test_mb_skcipher_data *data;
1062         struct crypto_skcipher *tfm;
1063         unsigned int i, j, iv_len;
1064         const int *b_size;
1065         const char *key;
1066         const char *e;
1067         char iv[128];
1068         int ret;
1069
1070         if (enc == ENCRYPT)
1071                 e = "encryption";
1072         else
1073                 e = "decryption";
1074
1075         data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
1076         if (!data)
1077                 return;
1078
1079         tfm = crypto_alloc_skcipher(algo, 0, 0);
1080         if (IS_ERR(tfm)) {
1081                 pr_err("failed to load transform for %s: %ld\n",
1082                         algo, PTR_ERR(tfm));
1083                 goto out_free_data;
1084         }
1085
1086         for (i = 0; i < num_mb; ++i)
1087                 if (testmgr_alloc_buf(data[i].xbuf)) {
1088                         while (i--)
1089                                 testmgr_free_buf(data[i].xbuf);
1090                         goto out_free_tfm;
1091                 }
1092
1093
1094         for (i = 0; i < num_mb; ++i)
1095                 if (testmgr_alloc_buf(data[i].xbuf)) {
1096                         while (i--)
1097                                 testmgr_free_buf(data[i].xbuf);
1098                         goto out_free_tfm;
1099                 }
1100
1101
1102         for (i = 0; i < num_mb; ++i) {
1103                 data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL);
1104                 if (!data[i].req) {
1105                         pr_err("alg: skcipher: Failed to allocate request for %s\n",
1106                                algo);
1107                         while (i--)
1108                                 skcipher_request_free(data[i].req);
1109                         goto out_free_xbuf;
1110                 }
1111         }
1112
1113         for (i = 0; i < num_mb; ++i) {
1114                 skcipher_request_set_callback(data[i].req,
1115                                               CRYPTO_TFM_REQ_MAY_BACKLOG,
1116                                               crypto_req_done, &data[i].wait);
1117                 crypto_init_wait(&data[i].wait);
1118         }
1119
1120         pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
1121                 get_driver_name(crypto_skcipher, tfm), e);
1122
1123         i = 0;
1124         do {
1125                 b_size = block_sizes;
1126                 do {
1127                         u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1128
1129                         if (bs > XBUFSIZE * PAGE_SIZE) {
1130                                 pr_err("template (%u) too big for buffer (%lu)\n",
1131                                        bs, XBUFSIZE * PAGE_SIZE);
1132                                 goto out;
1133                         }
1134
1135                         pr_info("test %u (%d bit key, %d byte blocks): ", i,
1136                                 *keysize * 8, bs);
1137
1138                         /* Set up tfm global state, i.e. the key */
1139
1140                         memset(tvmem[0], 0xff, PAGE_SIZE);
1141                         key = tvmem[0];
1142                         for (j = 0; j < tcount; j++) {
1143                                 if (template[j].klen == *keysize) {
1144                                         key = template[j].key;
1145                                         break;
1146                                 }
1147                         }
1148
1149                         crypto_skcipher_clear_flags(tfm, ~0);
1150
1151                         ret = crypto_skcipher_setkey(tfm, key, *keysize);
1152                         if (ret) {
1153                                 pr_err("setkey() failed flags=%x\n",
1154                                        crypto_skcipher_get_flags(tfm));
1155                                 goto out;
1156                         }
1157
1158                         iv_len = crypto_skcipher_ivsize(tfm);
1159                         if (iv_len)
1160                                 memset(&iv, 0xff, iv_len);
1161
1162                         /* Now setup per request stuff, i.e. buffers */
1163
1164                         for (j = 0; j < num_mb; ++j) {
1165                                 struct test_mb_skcipher_data *cur = &data[j];
1166                                 unsigned int k = bs;
1167                                 unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE);
1168                                 unsigned int p = 0;
1169
1170                                 sg_init_table(cur->sg, pages);
1171
1172                                 while (k > PAGE_SIZE) {
1173                                         sg_set_buf(cur->sg + p, cur->xbuf[p],
1174                                                    PAGE_SIZE);
1175                                         memset(cur->xbuf[p], 0xff, PAGE_SIZE);
1176                                         p++;
1177                                         k -= PAGE_SIZE;
1178                                 }
1179
1180                                 sg_set_buf(cur->sg + p, cur->xbuf[p], k);
1181                                 memset(cur->xbuf[p], 0xff, k);
1182
1183                                 skcipher_request_set_crypt(cur->req, cur->sg,
1184                                                            cur->sg, bs, iv);
1185                         }
1186
1187                         if (secs) {
1188                                 ret = test_mb_acipher_jiffies(data, enc,
1189                                                               bs, secs,
1190                                                               num_mb);
1191                                 cond_resched();
1192                         } else {
1193                                 ret = test_mb_acipher_cycles(data, enc,
1194                                                              bs, num_mb);
1195                         }
1196
1197                         if (ret) {
1198                                 pr_err("%s() failed flags=%x\n", e,
1199                                        crypto_skcipher_get_flags(tfm));
1200                                 break;
1201                         }
1202                         b_size++;
1203                         i++;
1204                 } while (*b_size);
1205                 keysize++;
1206         } while (*keysize);
1207
1208 out:
1209         for (i = 0; i < num_mb; ++i)
1210                 skcipher_request_free(data[i].req);
1211 out_free_xbuf:
1212         for (i = 0; i < num_mb; ++i)
1213                 testmgr_free_buf(data[i].xbuf);
1214 out_free_tfm:
1215         crypto_free_skcipher(tfm);
1216 out_free_data:
1217         kfree(data);
1218 }
1219
1220 static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
1221 {
1222         struct crypto_wait *wait = req->base.data;
1223
1224         return crypto_wait_req(ret, wait);
1225 }
1226
1227 static int test_acipher_jiffies(struct skcipher_request *req, int enc,
1228                                 int blen, int secs)
1229 {
1230         unsigned long start, end;
1231         int bcount;
1232         int ret;
1233
1234         for (start = jiffies, end = start + secs * HZ, bcount = 0;
1235              time_before(jiffies, end); bcount++) {
1236                 if (enc)
1237                         ret = do_one_acipher_op(req,
1238                                                 crypto_skcipher_encrypt(req));
1239                 else
1240                         ret = do_one_acipher_op(req,
1241                                                 crypto_skcipher_decrypt(req));
1242
1243                 if (ret)
1244                         return ret;
1245         }
1246
1247         pr_cont("%d operations in %d seconds (%llu bytes)\n",
1248                 bcount, secs, (u64)bcount * blen);
1249         return 0;
1250 }
1251
1252 static int test_acipher_cycles(struct skcipher_request *req, int enc,
1253                                int blen)
1254 {
1255         unsigned long cycles = 0;
1256         int ret = 0;
1257         int i;
1258
1259         /* Warm-up run. */
1260         for (i = 0; i < 4; i++) {
1261                 if (enc)
1262                         ret = do_one_acipher_op(req,
1263                                                 crypto_skcipher_encrypt(req));
1264                 else
1265                         ret = do_one_acipher_op(req,
1266                                                 crypto_skcipher_decrypt(req));
1267
1268                 if (ret)
1269                         goto out;
1270         }
1271
1272         /* The real thing. */
1273         for (i = 0; i < 8; i++) {
1274                 cycles_t start, end;
1275
1276                 start = get_cycles();
1277                 if (enc)
1278                         ret = do_one_acipher_op(req,
1279                                                 crypto_skcipher_encrypt(req));
1280                 else
1281                         ret = do_one_acipher_op(req,
1282                                                 crypto_skcipher_decrypt(req));
1283                 end = get_cycles();
1284
1285                 if (ret)
1286                         goto out;
1287
1288                 cycles += end - start;
1289         }
1290
1291 out:
1292         if (ret == 0)
1293                 pr_cont("1 operation in %lu cycles (%d bytes)\n",
1294                         (cycles + 4) / 8, blen);
1295
1296         return ret;
1297 }
1298
1299 static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
1300                                 struct cipher_speed_template *template,
1301                                 unsigned int tcount, u8 *keysize, bool async)
1302 {
1303         unsigned int ret, i, j, k, iv_len;
1304         struct crypto_wait wait;
1305         const char *key;
1306         char iv[128];
1307         struct skcipher_request *req;
1308         struct crypto_skcipher *tfm;
1309         const int *b_size;
1310         const char *e;
1311
1312         if (enc == ENCRYPT)
1313                 e = "encryption";
1314         else
1315                 e = "decryption";
1316
1317         crypto_init_wait(&wait);
1318
1319         tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
1320
1321         if (IS_ERR(tfm)) {
1322                 pr_err("failed to load transform for %s: %ld\n", algo,
1323                        PTR_ERR(tfm));
1324                 return;
1325         }
1326
1327         pr_info("\ntesting speed of %s %s (%s) %s\n", async ? "async" : "sync",
1328                 algo, get_driver_name(crypto_skcipher, tfm), e);
1329
1330         req = skcipher_request_alloc(tfm, GFP_KERNEL);
1331         if (!req) {
1332                 pr_err("tcrypt: skcipher: Failed to allocate request for %s\n",
1333                        algo);
1334                 goto out;
1335         }
1336
1337         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1338                                       crypto_req_done, &wait);
1339
1340         i = 0;
1341         do {
1342                 b_size = block_sizes;
1343
1344                 do {
1345                         u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1346                         struct scatterlist sg[TVMEMSIZE];
1347
1348                         if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
1349                                 pr_err("template (%u) too big for "
1350                                        "tvmem (%lu)\n", *keysize + bs,
1351                                        TVMEMSIZE * PAGE_SIZE);
1352                                 goto out_free_req;
1353                         }
1354
1355                         pr_info("test %u (%d bit key, %d byte blocks): ", i,
1356                                 *keysize * 8, bs);
1357
1358                         memset(tvmem[0], 0xff, PAGE_SIZE);
1359
1360                         /* set key, plain text and IV */
1361                         key = tvmem[0];
1362                         for (j = 0; j < tcount; j++) {
1363                                 if (template[j].klen == *keysize) {
1364                                         key = template[j].key;
1365                                         break;
1366                                 }
1367                         }
1368
1369                         crypto_skcipher_clear_flags(tfm, ~0);
1370
1371                         ret = crypto_skcipher_setkey(tfm, key, *keysize);
1372                         if (ret) {
1373                                 pr_err("setkey() failed flags=%x\n",
1374                                         crypto_skcipher_get_flags(tfm));
1375                                 goto out_free_req;
1376                         }
1377
1378                         k = *keysize + bs;
1379                         sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE));
1380
1381                         if (k > PAGE_SIZE) {
1382                                 sg_set_buf(sg, tvmem[0] + *keysize,
1383                                    PAGE_SIZE - *keysize);
1384                                 k -= PAGE_SIZE;
1385                                 j = 1;
1386                                 while (k > PAGE_SIZE) {
1387                                         sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
1388                                         memset(tvmem[j], 0xff, PAGE_SIZE);
1389                                         j++;
1390                                         k -= PAGE_SIZE;
1391                                 }
1392                                 sg_set_buf(sg + j, tvmem[j], k);
1393                                 memset(tvmem[j], 0xff, k);
1394                         } else {
1395                                 sg_set_buf(sg, tvmem[0] + *keysize, bs);
1396                         }
1397
1398                         iv_len = crypto_skcipher_ivsize(tfm);
1399                         if (iv_len)
1400                                 memset(&iv, 0xff, iv_len);
1401
1402                         skcipher_request_set_crypt(req, sg, sg, bs, iv);
1403
1404                         if (secs) {
1405                                 ret = test_acipher_jiffies(req, enc,
1406                                                            bs, secs);
1407                                 cond_resched();
1408                         } else {
1409                                 ret = test_acipher_cycles(req, enc,
1410                                                           bs);
1411                         }
1412
1413                         if (ret) {
1414                                 pr_err("%s() failed flags=%x\n", e,
1415                                        crypto_skcipher_get_flags(tfm));
1416                                 break;
1417                         }
1418                         b_size++;
1419                         i++;
1420                 } while (*b_size);
1421                 keysize++;
1422         } while (*keysize);
1423
1424 out_free_req:
1425         skcipher_request_free(req);
1426 out:
1427         crypto_free_skcipher(tfm);
1428 }
1429
1430 static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
1431                                struct cipher_speed_template *template,
1432                                unsigned int tcount, u8 *keysize)
1433 {
1434         return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1435                                    true);
1436 }
1437
1438 static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
1439                               struct cipher_speed_template *template,
1440                               unsigned int tcount, u8 *keysize)
1441 {
1442         return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1443                                    false);
1444 }
1445
1446 static inline int tcrypt_test(const char *alg)
1447 {
1448         int ret;
1449
1450         pr_debug("testing %s\n", alg);
1451
1452         ret = alg_test(alg, alg, 0, 0);
1453         /* non-fips algs return -EINVAL or -ECANCELED in fips mode */
1454         if (fips_enabled && (ret == -EINVAL || ret == -ECANCELED))
1455                 ret = 0;
1456         return ret;
1457 }
1458
1459 static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
1460 {
1461         int i;
1462         int ret = 0;
1463
1464         switch (m) {
1465         case 0:
1466                 if (alg) {
1467                         if (!crypto_has_alg(alg, type,
1468                                             mask ?: CRYPTO_ALG_TYPE_MASK))
1469                                 ret = -ENOENT;
1470                         break;
1471                 }
1472
1473                 for (i = 1; i < 200; i++)
1474                         ret += do_test(NULL, 0, 0, i, num_mb);
1475                 break;
1476
1477         case 1:
1478                 ret += tcrypt_test("md5");
1479                 break;
1480
1481         case 2:
1482                 ret += tcrypt_test("sha1");
1483                 break;
1484
1485         case 3:
1486                 ret += tcrypt_test("ecb(des)");
1487                 ret += tcrypt_test("cbc(des)");
1488                 ret += tcrypt_test("ctr(des)");
1489                 break;
1490
1491         case 4:
1492                 ret += tcrypt_test("ecb(des3_ede)");
1493                 ret += tcrypt_test("cbc(des3_ede)");
1494                 ret += tcrypt_test("ctr(des3_ede)");
1495                 break;
1496
1497         case 5:
1498                 ret += tcrypt_test("md4");
1499                 break;
1500
1501         case 6:
1502                 ret += tcrypt_test("sha256");
1503                 break;
1504
1505         case 7:
1506                 ret += tcrypt_test("ecb(blowfish)");
1507                 ret += tcrypt_test("cbc(blowfish)");
1508                 ret += tcrypt_test("ctr(blowfish)");
1509                 break;
1510
1511         case 8:
1512                 ret += tcrypt_test("ecb(twofish)");
1513                 ret += tcrypt_test("cbc(twofish)");
1514                 ret += tcrypt_test("ctr(twofish)");
1515                 ret += tcrypt_test("lrw(twofish)");
1516                 ret += tcrypt_test("xts(twofish)");
1517                 break;
1518
1519         case 9:
1520                 ret += tcrypt_test("ecb(serpent)");
1521                 ret += tcrypt_test("cbc(serpent)");
1522                 ret += tcrypt_test("ctr(serpent)");
1523                 ret += tcrypt_test("lrw(serpent)");
1524                 ret += tcrypt_test("xts(serpent)");
1525                 break;
1526
1527         case 10:
1528                 ret += tcrypt_test("ecb(aes)");
1529                 ret += tcrypt_test("cbc(aes)");
1530                 ret += tcrypt_test("lrw(aes)");
1531                 ret += tcrypt_test("xts(aes)");
1532                 ret += tcrypt_test("ctr(aes)");
1533                 ret += tcrypt_test("rfc3686(ctr(aes))");
1534                 ret += tcrypt_test("ofb(aes)");
1535                 ret += tcrypt_test("cfb(aes)");
1536                 ret += tcrypt_test("xctr(aes)");
1537                 break;
1538
1539         case 11:
1540                 ret += tcrypt_test("sha384");
1541                 break;
1542
1543         case 12:
1544                 ret += tcrypt_test("sha512");
1545                 break;
1546
1547         case 13:
1548                 ret += tcrypt_test("deflate");
1549                 break;
1550
1551         case 14:
1552                 ret += tcrypt_test("ecb(cast5)");
1553                 ret += tcrypt_test("cbc(cast5)");
1554                 ret += tcrypt_test("ctr(cast5)");
1555                 break;
1556
1557         case 15:
1558                 ret += tcrypt_test("ecb(cast6)");
1559                 ret += tcrypt_test("cbc(cast6)");
1560                 ret += tcrypt_test("ctr(cast6)");
1561                 ret += tcrypt_test("lrw(cast6)");
1562                 ret += tcrypt_test("xts(cast6)");
1563                 break;
1564
1565         case 16:
1566                 ret += tcrypt_test("ecb(arc4)");
1567                 break;
1568
1569         case 17:
1570                 ret += tcrypt_test("michael_mic");
1571                 break;
1572
1573         case 18:
1574                 ret += tcrypt_test("crc32c");
1575                 break;
1576
1577         case 19:
1578                 ret += tcrypt_test("ecb(tea)");
1579                 break;
1580
1581         case 20:
1582                 ret += tcrypt_test("ecb(xtea)");
1583                 break;
1584
1585         case 21:
1586                 ret += tcrypt_test("ecb(khazad)");
1587                 break;
1588
1589         case 22:
1590                 ret += tcrypt_test("wp512");
1591                 break;
1592
1593         case 23:
1594                 ret += tcrypt_test("wp384");
1595                 break;
1596
1597         case 24:
1598                 ret += tcrypt_test("wp256");
1599                 break;
1600
1601         case 26:
1602                 ret += tcrypt_test("ecb(anubis)");
1603                 ret += tcrypt_test("cbc(anubis)");
1604                 break;
1605
1606         case 30:
1607                 ret += tcrypt_test("ecb(xeta)");
1608                 break;
1609
1610         case 31:
1611                 ret += tcrypt_test("pcbc(fcrypt)");
1612                 break;
1613
1614         case 32:
1615                 ret += tcrypt_test("ecb(camellia)");
1616                 ret += tcrypt_test("cbc(camellia)");
1617                 ret += tcrypt_test("ctr(camellia)");
1618                 ret += tcrypt_test("lrw(camellia)");
1619                 ret += tcrypt_test("xts(camellia)");
1620                 break;
1621
1622         case 33:
1623                 ret += tcrypt_test("sha224");
1624                 break;
1625
1626         case 35:
1627                 ret += tcrypt_test("gcm(aes)");
1628                 break;
1629
1630         case 36:
1631                 ret += tcrypt_test("lzo");
1632                 break;
1633
1634         case 37:
1635                 ret += tcrypt_test("ccm(aes)");
1636                 break;
1637
1638         case 38:
1639                 ret += tcrypt_test("cts(cbc(aes))");
1640                 break;
1641
1642         case 39:
1643                 ret += tcrypt_test("xxhash64");
1644                 break;
1645
1646         case 40:
1647                 ret += tcrypt_test("rmd160");
1648                 break;
1649
1650         case 42:
1651                 ret += tcrypt_test("blake2b-512");
1652                 break;
1653
1654         case 43:
1655                 ret += tcrypt_test("ecb(seed)");
1656                 break;
1657
1658         case 45:
1659                 ret += tcrypt_test("rfc4309(ccm(aes))");
1660                 break;
1661
1662         case 46:
1663                 ret += tcrypt_test("ghash");
1664                 break;
1665
1666         case 47:
1667                 ret += tcrypt_test("crct10dif");
1668                 break;
1669
1670         case 48:
1671                 ret += tcrypt_test("sha3-224");
1672                 break;
1673
1674         case 49:
1675                 ret += tcrypt_test("sha3-256");
1676                 break;
1677
1678         case 50:
1679                 ret += tcrypt_test("sha3-384");
1680                 break;
1681
1682         case 51:
1683                 ret += tcrypt_test("sha3-512");
1684                 break;
1685
1686         case 52:
1687                 ret += tcrypt_test("sm3");
1688                 break;
1689
1690         case 53:
1691                 ret += tcrypt_test("streebog256");
1692                 break;
1693
1694         case 54:
1695                 ret += tcrypt_test("streebog512");
1696                 break;
1697
1698         case 55:
1699                 ret += tcrypt_test("gcm(sm4)");
1700                 break;
1701
1702         case 56:
1703                 ret += tcrypt_test("ccm(sm4)");
1704                 break;
1705
1706         case 57:
1707                 ret += tcrypt_test("polyval");
1708                 break;
1709
1710         case 58:
1711                 ret += tcrypt_test("gcm(aria)");
1712                 break;
1713
1714         case 100:
1715                 ret += tcrypt_test("hmac(md5)");
1716                 break;
1717
1718         case 101:
1719                 ret += tcrypt_test("hmac(sha1)");
1720                 break;
1721
1722         case 102:
1723                 ret += tcrypt_test("hmac(sha256)");
1724                 break;
1725
1726         case 103:
1727                 ret += tcrypt_test("hmac(sha384)");
1728                 break;
1729
1730         case 104:
1731                 ret += tcrypt_test("hmac(sha512)");
1732                 break;
1733
1734         case 105:
1735                 ret += tcrypt_test("hmac(sha224)");
1736                 break;
1737
1738         case 106:
1739                 ret += tcrypt_test("xcbc(aes)");
1740                 break;
1741
1742         case 108:
1743                 ret += tcrypt_test("hmac(rmd160)");
1744                 break;
1745
1746         case 109:
1747                 ret += tcrypt_test("vmac64(aes)");
1748                 break;
1749
1750         case 111:
1751                 ret += tcrypt_test("hmac(sha3-224)");
1752                 break;
1753
1754         case 112:
1755                 ret += tcrypt_test("hmac(sha3-256)");
1756                 break;
1757
1758         case 113:
1759                 ret += tcrypt_test("hmac(sha3-384)");
1760                 break;
1761
1762         case 114:
1763                 ret += tcrypt_test("hmac(sha3-512)");
1764                 break;
1765
1766         case 115:
1767                 ret += tcrypt_test("hmac(streebog256)");
1768                 break;
1769
1770         case 116:
1771                 ret += tcrypt_test("hmac(streebog512)");
1772                 break;
1773
1774         case 150:
1775                 ret += tcrypt_test("ansi_cprng");
1776                 break;
1777
1778         case 151:
1779                 ret += tcrypt_test("rfc4106(gcm(aes))");
1780                 break;
1781
1782         case 152:
1783                 ret += tcrypt_test("rfc4543(gcm(aes))");
1784                 break;
1785
1786         case 153:
1787                 ret += tcrypt_test("cmac(aes)");
1788                 break;
1789
1790         case 154:
1791                 ret += tcrypt_test("cmac(des3_ede)");
1792                 break;
1793
1794         case 155:
1795                 ret += tcrypt_test("authenc(hmac(sha1),cbc(aes))");
1796                 break;
1797
1798         case 156:
1799                 ret += tcrypt_test("authenc(hmac(md5),ecb(cipher_null))");
1800                 break;
1801
1802         case 157:
1803                 ret += tcrypt_test("authenc(hmac(sha1),ecb(cipher_null))");
1804                 break;
1805
1806         case 158:
1807                 ret += tcrypt_test("cbcmac(sm4)");
1808                 break;
1809
1810         case 159:
1811                 ret += tcrypt_test("cmac(sm4)");
1812                 break;
1813
1814         case 181:
1815                 ret += tcrypt_test("authenc(hmac(sha1),cbc(des))");
1816                 break;
1817         case 182:
1818                 ret += tcrypt_test("authenc(hmac(sha1),cbc(des3_ede))");
1819                 break;
1820         case 183:
1821                 ret += tcrypt_test("authenc(hmac(sha224),cbc(des))");
1822                 break;
1823         case 184:
1824                 ret += tcrypt_test("authenc(hmac(sha224),cbc(des3_ede))");
1825                 break;
1826         case 185:
1827                 ret += tcrypt_test("authenc(hmac(sha256),cbc(des))");
1828                 break;
1829         case 186:
1830                 ret += tcrypt_test("authenc(hmac(sha256),cbc(des3_ede))");
1831                 break;
1832         case 187:
1833                 ret += tcrypt_test("authenc(hmac(sha384),cbc(des))");
1834                 break;
1835         case 188:
1836                 ret += tcrypt_test("authenc(hmac(sha384),cbc(des3_ede))");
1837                 break;
1838         case 189:
1839                 ret += tcrypt_test("authenc(hmac(sha512),cbc(des))");
1840                 break;
1841         case 190:
1842                 ret += tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))");
1843                 break;
1844         case 191:
1845                 ret += tcrypt_test("ecb(sm4)");
1846                 ret += tcrypt_test("cbc(sm4)");
1847                 ret += tcrypt_test("cfb(sm4)");
1848                 ret += tcrypt_test("ctr(sm4)");
1849                 break;
1850         case 192:
1851                 ret += tcrypt_test("ecb(aria)");
1852                 ret += tcrypt_test("cbc(aria)");
1853                 ret += tcrypt_test("cfb(aria)");
1854                 ret += tcrypt_test("ctr(aria)");
1855                 break;
1856         case 200:
1857                 test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
1858                                 speed_template_16_24_32);
1859                 test_cipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
1860                                 speed_template_16_24_32);
1861                 test_cipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
1862                                 speed_template_16_24_32);
1863                 test_cipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
1864                                 speed_template_16_24_32);
1865                 test_cipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
1866                                 speed_template_32_40_48);
1867                 test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
1868                                 speed_template_32_40_48);
1869                 test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
1870                                 speed_template_32_64);
1871                 test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
1872                                 speed_template_32_64);
1873                 test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
1874                                 speed_template_16_24_32);
1875                 test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
1876                                 speed_template_16_24_32);
1877                 test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
1878                                 speed_template_16_24_32);
1879                 test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
1880                                 speed_template_16_24_32);
1881                 test_cipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
1882                                 speed_template_16_24_32);
1883                 test_cipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
1884                                 speed_template_16_24_32);
1885                 break;
1886
1887         case 201:
1888                 test_cipher_speed("ecb(des3_ede)", ENCRYPT, sec,
1889                                 des3_speed_template, DES3_SPEED_VECTORS,
1890                                 speed_template_24);
1891                 test_cipher_speed("ecb(des3_ede)", DECRYPT, sec,
1892                                 des3_speed_template, DES3_SPEED_VECTORS,
1893                                 speed_template_24);
1894                 test_cipher_speed("cbc(des3_ede)", ENCRYPT, sec,
1895                                 des3_speed_template, DES3_SPEED_VECTORS,
1896                                 speed_template_24);
1897                 test_cipher_speed("cbc(des3_ede)", DECRYPT, sec,
1898                                 des3_speed_template, DES3_SPEED_VECTORS,
1899                                 speed_template_24);
1900                 test_cipher_speed("ctr(des3_ede)", ENCRYPT, sec,
1901                                 des3_speed_template, DES3_SPEED_VECTORS,
1902                                 speed_template_24);
1903                 test_cipher_speed("ctr(des3_ede)", DECRYPT, sec,
1904                                 des3_speed_template, DES3_SPEED_VECTORS,
1905                                 speed_template_24);
1906                 break;
1907
1908         case 202:
1909                 test_cipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
1910                                 speed_template_16_24_32);
1911                 test_cipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
1912                                 speed_template_16_24_32);
1913                 test_cipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
1914                                 speed_template_16_24_32);
1915                 test_cipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
1916                                 speed_template_16_24_32);
1917                 test_cipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
1918                                 speed_template_16_24_32);
1919                 test_cipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
1920                                 speed_template_16_24_32);
1921                 test_cipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
1922                                 speed_template_32_40_48);
1923                 test_cipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
1924                                 speed_template_32_40_48);
1925                 test_cipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
1926                                 speed_template_32_48_64);
1927                 test_cipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
1928                                 speed_template_32_48_64);
1929                 break;
1930
1931         case 203:
1932                 test_cipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
1933                                   speed_template_8_32);
1934                 test_cipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
1935                                   speed_template_8_32);
1936                 test_cipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
1937                                   speed_template_8_32);
1938                 test_cipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
1939                                   speed_template_8_32);
1940                 test_cipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
1941                                   speed_template_8_32);
1942                 test_cipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
1943                                   speed_template_8_32);
1944                 break;
1945
1946         case 204:
1947                 test_cipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
1948                                   speed_template_8);
1949                 test_cipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
1950                                   speed_template_8);
1951                 test_cipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
1952                                   speed_template_8);
1953                 test_cipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
1954                                   speed_template_8);
1955                 break;
1956
1957         case 205:
1958                 test_cipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
1959                                 speed_template_16_24_32);
1960                 test_cipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
1961                                 speed_template_16_24_32);
1962                 test_cipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
1963                                 speed_template_16_24_32);
1964                 test_cipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
1965                                 speed_template_16_24_32);
1966                 test_cipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
1967                                 speed_template_16_24_32);
1968                 test_cipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
1969                                 speed_template_16_24_32);
1970                 test_cipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
1971                                 speed_template_32_40_48);
1972                 test_cipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
1973                                 speed_template_32_40_48);
1974                 test_cipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
1975                                 speed_template_32_48_64);
1976                 test_cipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
1977                                 speed_template_32_48_64);
1978                 break;
1979
1980         case 207:
1981                 test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
1982                                   speed_template_16_32);
1983                 test_cipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
1984                                   speed_template_16_32);
1985                 test_cipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
1986                                   speed_template_16_32);
1987                 test_cipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
1988                                   speed_template_16_32);
1989                 test_cipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
1990                                   speed_template_16_32);
1991                 test_cipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
1992                                   speed_template_16_32);
1993                 test_cipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
1994                                   speed_template_32_48);
1995                 test_cipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
1996                                   speed_template_32_48);
1997                 test_cipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
1998                                   speed_template_32_64);
1999                 test_cipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2000                                   speed_template_32_64);
2001                 break;
2002
2003         case 208:
2004                 test_cipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2005                                   speed_template_8);
2006                 break;
2007
2008         case 209:
2009                 test_cipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2010                                   speed_template_8_16);
2011                 test_cipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2012                                   speed_template_8_16);
2013                 test_cipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2014                                   speed_template_8_16);
2015                 test_cipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2016                                   speed_template_8_16);
2017                 test_cipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2018                                   speed_template_8_16);
2019                 test_cipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2020                                   speed_template_8_16);
2021                 break;
2022
2023         case 210:
2024                 test_cipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2025                                   speed_template_16_32);
2026                 test_cipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2027                                   speed_template_16_32);
2028                 test_cipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2029                                   speed_template_16_32);
2030                 test_cipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2031                                   speed_template_16_32);
2032                 test_cipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2033                                   speed_template_16_32);
2034                 test_cipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2035                                   speed_template_16_32);
2036                 test_cipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2037                                   speed_template_32_48);
2038                 test_cipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2039                                   speed_template_32_48);
2040                 test_cipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2041                                   speed_template_32_64);
2042                 test_cipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2043                                   speed_template_32_64);
2044                 break;
2045
2046         case 211:
2047                 test_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec,
2048                                 NULL, 0, 16, 16, aead_speed_template_20);
2049                 test_aead_speed("gcm(aes)", ENCRYPT, sec,
2050                                 NULL, 0, 16, 8, speed_template_16_24_32);
2051                 test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec,
2052                                 NULL, 0, 16, 16, aead_speed_template_20);
2053                 test_aead_speed("gcm(aes)", DECRYPT, sec,
2054                                 NULL, 0, 16, 8, speed_template_16_24_32);
2055                 break;
2056
2057         case 212:
2058                 test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec,
2059                                 NULL, 0, 16, 16, aead_speed_template_19);
2060                 test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec,
2061                                 NULL, 0, 16, 16, aead_speed_template_19);
2062                 break;
2063
2064         case 213:
2065                 test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec,
2066                                 NULL, 0, 16, 8, aead_speed_template_36);
2067                 test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec,
2068                                 NULL, 0, 16, 8, aead_speed_template_36);
2069                 break;
2070
2071         case 214:
2072                 test_cipher_speed("chacha20", ENCRYPT, sec, NULL, 0,
2073                                   speed_template_32);
2074                 break;
2075
2076         case 215:
2077                 test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL,
2078                                    0, 16, 16, aead_speed_template_20, num_mb);
2079                 test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8,
2080                                    speed_template_16_24_32, num_mb);
2081                 test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL,
2082                                    0, 16, 16, aead_speed_template_20, num_mb);
2083                 test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8,
2084                                    speed_template_16_24_32, num_mb);
2085                 break;
2086
2087         case 216:
2088                 test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0,
2089                                    16, 16, aead_speed_template_19, num_mb);
2090                 test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0,
2091                                    16, 16, aead_speed_template_19, num_mb);
2092                 break;
2093
2094         case 217:
2095                 test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT,
2096                                    sec, NULL, 0, 16, 8, aead_speed_template_36,
2097                                    num_mb);
2098                 test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT,
2099                                    sec, NULL, 0, 16, 8, aead_speed_template_36,
2100                                    num_mb);
2101                 break;
2102
2103         case 218:
2104                 test_cipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2105                                 speed_template_16);
2106                 test_cipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2107                                 speed_template_16);
2108                 test_cipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2109                                 speed_template_16);
2110                 test_cipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2111                                 speed_template_16);
2112                 test_cipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0,
2113                                 speed_template_16);
2114                 test_cipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0,
2115                                 speed_template_16);
2116                 test_cipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2117                                 speed_template_16);
2118                 test_cipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2119                                 speed_template_16);
2120                 break;
2121
2122         case 219:
2123                 test_cipher_speed("adiantum(xchacha12,aes)", ENCRYPT, sec, NULL,
2124                                   0, speed_template_32);
2125                 test_cipher_speed("adiantum(xchacha12,aes)", DECRYPT, sec, NULL,
2126                                   0, speed_template_32);
2127                 test_cipher_speed("adiantum(xchacha20,aes)", ENCRYPT, sec, NULL,
2128                                   0, speed_template_32);
2129                 test_cipher_speed("adiantum(xchacha20,aes)", DECRYPT, sec, NULL,
2130                                   0, speed_template_32);
2131                 break;
2132
2133         case 220:
2134                 test_acipher_speed("essiv(cbc(aes),sha256)",
2135                                   ENCRYPT, sec, NULL, 0,
2136                                   speed_template_16_24_32);
2137                 test_acipher_speed("essiv(cbc(aes),sha256)",
2138                                   DECRYPT, sec, NULL, 0,
2139                                   speed_template_16_24_32);
2140                 break;
2141
2142         case 221:
2143                 test_aead_speed("aegis128", ENCRYPT, sec,
2144                                 NULL, 0, 16, 8, speed_template_16);
2145                 test_aead_speed("aegis128", DECRYPT, sec,
2146                                 NULL, 0, 16, 8, speed_template_16);
2147                 break;
2148
2149         case 222:
2150                 test_aead_speed("gcm(sm4)", ENCRYPT, sec,
2151                                 NULL, 0, 16, 8, speed_template_16);
2152                 test_aead_speed("gcm(sm4)", DECRYPT, sec,
2153                                 NULL, 0, 16, 8, speed_template_16);
2154                 break;
2155
2156         case 223:
2157                 test_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec,
2158                                 NULL, 0, 16, 16, aead_speed_template_19);
2159                 test_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec,
2160                                 NULL, 0, 16, 16, aead_speed_template_19);
2161                 break;
2162
2163         case 224:
2164                 test_mb_aead_speed("gcm(sm4)", ENCRYPT, sec, NULL, 0, 16, 8,
2165                                    speed_template_16, num_mb);
2166                 test_mb_aead_speed("gcm(sm4)", DECRYPT, sec, NULL, 0, 16, 8,
2167                                    speed_template_16, num_mb);
2168                 break;
2169
2170         case 225:
2171                 test_mb_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec, NULL, 0,
2172                                    16, 16, aead_speed_template_19, num_mb);
2173                 test_mb_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec, NULL, 0,
2174                                    16, 16, aead_speed_template_19, num_mb);
2175                 break;
2176
2177         case 226:
2178                 test_cipher_speed("hctr2(aes)", ENCRYPT, sec, NULL,
2179                                   0, speed_template_32);
2180                 break;
2181
2182         case 227:
2183                 test_cipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2184                                   speed_template_16_24_32);
2185                 test_cipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2186                                   speed_template_16_24_32);
2187                 test_cipher_speed("cbc(aria)", ENCRYPT, sec, NULL, 0,
2188                                   speed_template_16_24_32);
2189                 test_cipher_speed("cbc(aria)", DECRYPT, sec, NULL, 0,
2190                                   speed_template_16_24_32);
2191                 test_cipher_speed("cfb(aria)", ENCRYPT, sec, NULL, 0,
2192                                   speed_template_16_24_32);
2193                 test_cipher_speed("cfb(aria)", DECRYPT, sec, NULL, 0,
2194                                   speed_template_16_24_32);
2195                 test_cipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2196                                   speed_template_16_24_32);
2197                 test_cipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2198                                   speed_template_16_24_32);
2199                 break;
2200
2201         case 228:
2202                 test_aead_speed("gcm(aria)", ENCRYPT, sec,
2203                                 NULL, 0, 16, 8, speed_template_16_24_32);
2204                 test_aead_speed("gcm(aria)", DECRYPT, sec,
2205                                 NULL, 0, 16, 8, speed_template_16_24_32);
2206                 break;
2207
2208         case 229:
2209                 test_mb_aead_speed("gcm(aria)", ENCRYPT, sec, NULL, 0, 16, 8,
2210                                    speed_template_16, num_mb);
2211                 test_mb_aead_speed("gcm(aria)", DECRYPT, sec, NULL, 0, 16, 8,
2212                                    speed_template_16, num_mb);
2213                 break;
2214
2215         case 300:
2216                 if (alg) {
2217                         test_hash_speed(alg, sec, generic_hash_speed_template);
2218                         break;
2219                 }
2220                 fallthrough;
2221         case 301:
2222                 test_hash_speed("md4", sec, generic_hash_speed_template);
2223                 if (mode > 300 && mode < 400) break;
2224                 fallthrough;
2225         case 302:
2226                 test_hash_speed("md5", sec, generic_hash_speed_template);
2227                 if (mode > 300 && mode < 400) break;
2228                 fallthrough;
2229         case 303:
2230                 test_hash_speed("sha1", sec, generic_hash_speed_template);
2231                 if (mode > 300 && mode < 400) break;
2232                 fallthrough;
2233         case 304:
2234                 test_hash_speed("sha256", sec, generic_hash_speed_template);
2235                 if (mode > 300 && mode < 400) break;
2236                 fallthrough;
2237         case 305:
2238                 test_hash_speed("sha384", sec, generic_hash_speed_template);
2239                 if (mode > 300 && mode < 400) break;
2240                 fallthrough;
2241         case 306:
2242                 test_hash_speed("sha512", sec, generic_hash_speed_template);
2243                 if (mode > 300 && mode < 400) break;
2244                 fallthrough;
2245         case 307:
2246                 test_hash_speed("wp256", sec, generic_hash_speed_template);
2247                 if (mode > 300 && mode < 400) break;
2248                 fallthrough;
2249         case 308:
2250                 test_hash_speed("wp384", sec, generic_hash_speed_template);
2251                 if (mode > 300 && mode < 400) break;
2252                 fallthrough;
2253         case 309:
2254                 test_hash_speed("wp512", sec, generic_hash_speed_template);
2255                 if (mode > 300 && mode < 400) break;
2256                 fallthrough;
2257         case 313:
2258                 test_hash_speed("sha224", sec, generic_hash_speed_template);
2259                 if (mode > 300 && mode < 400) break;
2260                 fallthrough;
2261         case 314:
2262                 test_hash_speed("xxhash64", sec, generic_hash_speed_template);
2263                 if (mode > 300 && mode < 400) break;
2264                 fallthrough;
2265         case 315:
2266                 test_hash_speed("rmd160", sec, generic_hash_speed_template);
2267                 if (mode > 300 && mode < 400) break;
2268                 fallthrough;
2269         case 317:
2270                 test_hash_speed("blake2b-512", sec, generic_hash_speed_template);
2271                 if (mode > 300 && mode < 400) break;
2272                 fallthrough;
2273         case 318:
2274                 klen = 16;
2275                 test_hash_speed("ghash", sec, generic_hash_speed_template);
2276                 if (mode > 300 && mode < 400) break;
2277                 fallthrough;
2278         case 319:
2279                 test_hash_speed("crc32c", sec, generic_hash_speed_template);
2280                 if (mode > 300 && mode < 400) break;
2281                 fallthrough;
2282         case 320:
2283                 test_hash_speed("crct10dif", sec, generic_hash_speed_template);
2284                 if (mode > 300 && mode < 400) break;
2285                 fallthrough;
2286         case 321:
2287                 test_hash_speed("poly1305", sec, poly1305_speed_template);
2288                 if (mode > 300 && mode < 400) break;
2289                 fallthrough;
2290         case 322:
2291                 test_hash_speed("sha3-224", sec, generic_hash_speed_template);
2292                 if (mode > 300 && mode < 400) break;
2293                 fallthrough;
2294         case 323:
2295                 test_hash_speed("sha3-256", sec, generic_hash_speed_template);
2296                 if (mode > 300 && mode < 400) break;
2297                 fallthrough;
2298         case 324:
2299                 test_hash_speed("sha3-384", sec, generic_hash_speed_template);
2300                 if (mode > 300 && mode < 400) break;
2301                 fallthrough;
2302         case 325:
2303                 test_hash_speed("sha3-512", sec, generic_hash_speed_template);
2304                 if (mode > 300 && mode < 400) break;
2305                 fallthrough;
2306         case 326:
2307                 test_hash_speed("sm3", sec, generic_hash_speed_template);
2308                 if (mode > 300 && mode < 400) break;
2309                 fallthrough;
2310         case 327:
2311                 test_hash_speed("streebog256", sec,
2312                                 generic_hash_speed_template);
2313                 if (mode > 300 && mode < 400) break;
2314                 fallthrough;
2315         case 328:
2316                 test_hash_speed("streebog512", sec,
2317                                 generic_hash_speed_template);
2318                 if (mode > 300 && mode < 400) break;
2319                 fallthrough;
2320         case 399:
2321                 break;
2322
2323         case 400:
2324                 if (alg) {
2325                         test_ahash_speed(alg, sec, generic_hash_speed_template);
2326                         break;
2327                 }
2328                 fallthrough;
2329         case 401:
2330                 test_ahash_speed("md4", sec, generic_hash_speed_template);
2331                 if (mode > 400 && mode < 500) break;
2332                 fallthrough;
2333         case 402:
2334                 test_ahash_speed("md5", sec, generic_hash_speed_template);
2335                 if (mode > 400 && mode < 500) break;
2336                 fallthrough;
2337         case 403:
2338                 test_ahash_speed("sha1", sec, generic_hash_speed_template);
2339                 if (mode > 400 && mode < 500) break;
2340                 fallthrough;
2341         case 404:
2342                 test_ahash_speed("sha256", sec, generic_hash_speed_template);
2343                 if (mode > 400 && mode < 500) break;
2344                 fallthrough;
2345         case 405:
2346                 test_ahash_speed("sha384", sec, generic_hash_speed_template);
2347                 if (mode > 400 && mode < 500) break;
2348                 fallthrough;
2349         case 406:
2350                 test_ahash_speed("sha512", sec, generic_hash_speed_template);
2351                 if (mode > 400 && mode < 500) break;
2352                 fallthrough;
2353         case 407:
2354                 test_ahash_speed("wp256", sec, generic_hash_speed_template);
2355                 if (mode > 400 && mode < 500) break;
2356                 fallthrough;
2357         case 408:
2358                 test_ahash_speed("wp384", sec, generic_hash_speed_template);
2359                 if (mode > 400 && mode < 500) break;
2360                 fallthrough;
2361         case 409:
2362                 test_ahash_speed("wp512", sec, generic_hash_speed_template);
2363                 if (mode > 400 && mode < 500) break;
2364                 fallthrough;
2365         case 413:
2366                 test_ahash_speed("sha224", sec, generic_hash_speed_template);
2367                 if (mode > 400 && mode < 500) break;
2368                 fallthrough;
2369         case 414:
2370                 test_ahash_speed("xxhash64", sec, generic_hash_speed_template);
2371                 if (mode > 400 && mode < 500) break;
2372                 fallthrough;
2373         case 415:
2374                 test_ahash_speed("rmd160", sec, generic_hash_speed_template);
2375                 if (mode > 400 && mode < 500) break;
2376                 fallthrough;
2377         case 417:
2378                 test_ahash_speed("blake2b-512", sec, generic_hash_speed_template);
2379                 if (mode > 400 && mode < 500) break;
2380                 fallthrough;
2381         case 418:
2382                 test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
2383                 if (mode > 400 && mode < 500) break;
2384                 fallthrough;
2385         case 419:
2386                 test_ahash_speed("sha3-256", sec, generic_hash_speed_template);
2387                 if (mode > 400 && mode < 500) break;
2388                 fallthrough;
2389         case 420:
2390                 test_ahash_speed("sha3-384", sec, generic_hash_speed_template);
2391                 if (mode > 400 && mode < 500) break;
2392                 fallthrough;
2393         case 421:
2394                 test_ahash_speed("sha3-512", sec, generic_hash_speed_template);
2395                 if (mode > 400 && mode < 500) break;
2396                 fallthrough;
2397         case 422:
2398                 test_ahash_speed("sm3", sec, generic_hash_speed_template);
2399                 if (mode > 400 && mode < 500) break;
2400                 fallthrough;
2401         case 499:
2402                 break;
2403
2404         case 500:
2405                 test_acipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2406                                    speed_template_16_24_32);
2407                 test_acipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2408                                    speed_template_16_24_32);
2409                 test_acipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2410                                    speed_template_16_24_32);
2411                 test_acipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2412                                    speed_template_16_24_32);
2413                 test_acipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2414                                    speed_template_32_40_48);
2415                 test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2416                                    speed_template_32_40_48);
2417                 test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2418                                    speed_template_32_64);
2419                 test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2420                                    speed_template_32_64);
2421                 test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2422                                    speed_template_16_24_32);
2423                 test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2424                                    speed_template_16_24_32);
2425                 test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2426                                    speed_template_16_24_32);
2427                 test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2428                                    speed_template_16_24_32);
2429                 test_acipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
2430                                    speed_template_16_24_32);
2431                 test_acipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
2432                                    speed_template_16_24_32);
2433                 test_acipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
2434                                    speed_template_16_24_32);
2435                 test_acipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
2436                                    speed_template_16_24_32);
2437                 test_acipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0,
2438                                    speed_template_20_28_36);
2439                 test_acipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, 0,
2440                                    speed_template_20_28_36);
2441                 break;
2442
2443         case 501:
2444                 test_acipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2445                                    des3_speed_template, DES3_SPEED_VECTORS,
2446                                    speed_template_24);
2447                 test_acipher_speed("ecb(des3_ede)", DECRYPT, sec,
2448                                    des3_speed_template, DES3_SPEED_VECTORS,
2449                                    speed_template_24);
2450                 test_acipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2451                                    des3_speed_template, DES3_SPEED_VECTORS,
2452                                    speed_template_24);
2453                 test_acipher_speed("cbc(des3_ede)", DECRYPT, sec,
2454                                    des3_speed_template, DES3_SPEED_VECTORS,
2455                                    speed_template_24);
2456                 test_acipher_speed("cfb(des3_ede)", ENCRYPT, sec,
2457                                    des3_speed_template, DES3_SPEED_VECTORS,
2458                                    speed_template_24);
2459                 test_acipher_speed("cfb(des3_ede)", DECRYPT, sec,
2460                                    des3_speed_template, DES3_SPEED_VECTORS,
2461                                    speed_template_24);
2462                 test_acipher_speed("ofb(des3_ede)", ENCRYPT, sec,
2463                                    des3_speed_template, DES3_SPEED_VECTORS,
2464                                    speed_template_24);
2465                 test_acipher_speed("ofb(des3_ede)", DECRYPT, sec,
2466                                    des3_speed_template, DES3_SPEED_VECTORS,
2467                                    speed_template_24);
2468                 break;
2469
2470         case 502:
2471                 test_acipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2472                                    speed_template_8);
2473                 test_acipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2474                                    speed_template_8);
2475                 test_acipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2476                                    speed_template_8);
2477                 test_acipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2478                                    speed_template_8);
2479                 test_acipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
2480                                    speed_template_8);
2481                 test_acipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
2482                                    speed_template_8);
2483                 test_acipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
2484                                    speed_template_8);
2485                 test_acipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
2486                                    speed_template_8);
2487                 break;
2488
2489         case 503:
2490                 test_acipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2491                                    speed_template_16_32);
2492                 test_acipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2493                                    speed_template_16_32);
2494                 test_acipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2495                                    speed_template_16_32);
2496                 test_acipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2497                                    speed_template_16_32);
2498                 test_acipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2499                                    speed_template_16_32);
2500                 test_acipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2501                                    speed_template_16_32);
2502                 test_acipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2503                                    speed_template_32_48);
2504                 test_acipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2505                                    speed_template_32_48);
2506                 test_acipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2507                                    speed_template_32_64);
2508                 test_acipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2509                                    speed_template_32_64);
2510                 break;
2511
2512         case 504:
2513                 test_acipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2514                                    speed_template_16_24_32);
2515                 test_acipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2516                                    speed_template_16_24_32);
2517                 test_acipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2518                                    speed_template_16_24_32);
2519                 test_acipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2520                                    speed_template_16_24_32);
2521                 test_acipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2522                                    speed_template_16_24_32);
2523                 test_acipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2524                                    speed_template_16_24_32);
2525                 test_acipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2526                                    speed_template_32_40_48);
2527                 test_acipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2528                                    speed_template_32_40_48);
2529                 test_acipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2530                                    speed_template_32_48_64);
2531                 test_acipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2532                                    speed_template_32_48_64);
2533                 break;
2534
2535         case 505:
2536                 test_acipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2537                                    speed_template_8);
2538                 break;
2539
2540         case 506:
2541                 test_acipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2542                                    speed_template_8_16);
2543                 test_acipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2544                                    speed_template_8_16);
2545                 test_acipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2546                                    speed_template_8_16);
2547                 test_acipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2548                                    speed_template_8_16);
2549                 test_acipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2550                                    speed_template_8_16);
2551                 test_acipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2552                                    speed_template_8_16);
2553                 break;
2554
2555         case 507:
2556                 test_acipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2557                                    speed_template_16_32);
2558                 test_acipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2559                                    speed_template_16_32);
2560                 test_acipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2561                                    speed_template_16_32);
2562                 test_acipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2563                                    speed_template_16_32);
2564                 test_acipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2565                                    speed_template_16_32);
2566                 test_acipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2567                                    speed_template_16_32);
2568                 test_acipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2569                                    speed_template_32_48);
2570                 test_acipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2571                                    speed_template_32_48);
2572                 test_acipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2573                                    speed_template_32_64);
2574                 test_acipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2575                                    speed_template_32_64);
2576                 break;
2577
2578         case 508:
2579                 test_acipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2580                                    speed_template_16_32);
2581                 test_acipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2582                                    speed_template_16_32);
2583                 test_acipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2584                                    speed_template_16_32);
2585                 test_acipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2586                                    speed_template_16_32);
2587                 test_acipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2588                                    speed_template_16_32);
2589                 test_acipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2590                                    speed_template_16_32);
2591                 test_acipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2592                                    speed_template_32_48);
2593                 test_acipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2594                                    speed_template_32_48);
2595                 test_acipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2596                                    speed_template_32_64);
2597                 test_acipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2598                                    speed_template_32_64);
2599                 break;
2600
2601         case 509:
2602                 test_acipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2603                                    speed_template_8_32);
2604                 test_acipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2605                                    speed_template_8_32);
2606                 test_acipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2607                                    speed_template_8_32);
2608                 test_acipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2609                                    speed_template_8_32);
2610                 test_acipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2611                                    speed_template_8_32);
2612                 test_acipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2613                                    speed_template_8_32);
2614                 break;
2615
2616         case 518:
2617                 test_acipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2618                                 speed_template_16);
2619                 test_acipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2620                                 speed_template_16);
2621                 test_acipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2622                                 speed_template_16);
2623                 test_acipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2624                                 speed_template_16);
2625                 test_acipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0,
2626                                 speed_template_16);
2627                 test_acipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0,
2628                                 speed_template_16);
2629                 test_acipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2630                                 speed_template_16);
2631                 test_acipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2632                                 speed_template_16);
2633                 break;
2634
2635         case 519:
2636                 test_acipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2637                                    speed_template_16_24_32);
2638                 test_acipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2639                                    speed_template_16_24_32);
2640                 test_acipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2641                                    speed_template_16_24_32);
2642                 test_acipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2643                                    speed_template_16_24_32);
2644                 break;
2645
2646         case 600:
2647                 test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2648                                        speed_template_16_24_32, num_mb);
2649                 test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2650                                        speed_template_16_24_32, num_mb);
2651                 test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2652                                        speed_template_16_24_32, num_mb);
2653                 test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2654                                        speed_template_16_24_32, num_mb);
2655                 test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2656                                        speed_template_32_40_48, num_mb);
2657                 test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2658                                        speed_template_32_40_48, num_mb);
2659                 test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2660                                        speed_template_32_64, num_mb);
2661                 test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2662                                        speed_template_32_64, num_mb);
2663                 test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2664                                        speed_template_16_24_32, num_mb);
2665                 test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2666                                        speed_template_16_24_32, num_mb);
2667                 test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2668                                        speed_template_16_24_32, num_mb);
2669                 test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2670                                        speed_template_16_24_32, num_mb);
2671                 test_mb_skcipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
2672                                        speed_template_16_24_32, num_mb);
2673                 test_mb_skcipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
2674                                        speed_template_16_24_32, num_mb);
2675                 test_mb_skcipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
2676                                        speed_template_16_24_32, num_mb);
2677                 test_mb_skcipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
2678                                        speed_template_16_24_32, num_mb);
2679                 test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL,
2680                                        0, speed_template_20_28_36, num_mb);
2681                 test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL,
2682                                        0, speed_template_20_28_36, num_mb);
2683                 break;
2684
2685         case 601:
2686                 test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2687                                        des3_speed_template, DES3_SPEED_VECTORS,
2688                                        speed_template_24, num_mb);
2689                 test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec,
2690                                        des3_speed_template, DES3_SPEED_VECTORS,
2691                                        speed_template_24, num_mb);
2692                 test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2693                                        des3_speed_template, DES3_SPEED_VECTORS,
2694                                        speed_template_24, num_mb);
2695                 test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec,
2696                                        des3_speed_template, DES3_SPEED_VECTORS,
2697                                        speed_template_24, num_mb);
2698                 test_mb_skcipher_speed("cfb(des3_ede)", ENCRYPT, sec,
2699                                        des3_speed_template, DES3_SPEED_VECTORS,
2700                                        speed_template_24, num_mb);
2701                 test_mb_skcipher_speed("cfb(des3_ede)", DECRYPT, sec,
2702                                        des3_speed_template, DES3_SPEED_VECTORS,
2703                                        speed_template_24, num_mb);
2704                 test_mb_skcipher_speed("ofb(des3_ede)", ENCRYPT, sec,
2705                                        des3_speed_template, DES3_SPEED_VECTORS,
2706                                        speed_template_24, num_mb);
2707                 test_mb_skcipher_speed("ofb(des3_ede)", DECRYPT, sec,
2708                                        des3_speed_template, DES3_SPEED_VECTORS,
2709                                        speed_template_24, num_mb);
2710                 break;
2711
2712         case 602:
2713                 test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2714                                        speed_template_8, num_mb);
2715                 test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2716                                        speed_template_8, num_mb);
2717                 test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2718                                        speed_template_8, num_mb);
2719                 test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2720                                        speed_template_8, num_mb);
2721                 test_mb_skcipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
2722                                        speed_template_8, num_mb);
2723                 test_mb_skcipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
2724                                        speed_template_8, num_mb);
2725                 test_mb_skcipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
2726                                        speed_template_8, num_mb);
2727                 test_mb_skcipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
2728                                        speed_template_8, num_mb);
2729                 break;
2730
2731         case 603:
2732                 test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2733                                        speed_template_16_32, num_mb);
2734                 test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2735                                        speed_template_16_32, num_mb);
2736                 test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2737                                        speed_template_16_32, num_mb);
2738                 test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2739                                        speed_template_16_32, num_mb);
2740                 test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2741                                        speed_template_16_32, num_mb);
2742                 test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2743                                        speed_template_16_32, num_mb);
2744                 test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2745                                        speed_template_32_48, num_mb);
2746                 test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2747                                        speed_template_32_48, num_mb);
2748                 test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2749                                        speed_template_32_64, num_mb);
2750                 test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2751                                        speed_template_32_64, num_mb);
2752                 break;
2753
2754         case 604:
2755                 test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2756                                        speed_template_16_24_32, num_mb);
2757                 test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2758                                        speed_template_16_24_32, num_mb);
2759                 test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2760                                        speed_template_16_24_32, num_mb);
2761                 test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2762                                        speed_template_16_24_32, num_mb);
2763                 test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2764                                        speed_template_16_24_32, num_mb);
2765                 test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2766                                        speed_template_16_24_32, num_mb);
2767                 test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2768                                        speed_template_32_40_48, num_mb);
2769                 test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2770                                        speed_template_32_40_48, num_mb);
2771                 test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2772                                        speed_template_32_48_64, num_mb);
2773                 test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2774                                        speed_template_32_48_64, num_mb);
2775                 break;
2776
2777         case 605:
2778                 test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2779                                        speed_template_8, num_mb);
2780                 break;
2781
2782         case 606:
2783                 test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2784                                        speed_template_8_16, num_mb);
2785                 test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2786                                        speed_template_8_16, num_mb);
2787                 test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2788                                        speed_template_8_16, num_mb);
2789                 test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2790                                        speed_template_8_16, num_mb);
2791                 test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2792                                        speed_template_8_16, num_mb);
2793                 test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2794                                        speed_template_8_16, num_mb);
2795                 break;
2796
2797         case 607:
2798                 test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2799                                        speed_template_16_32, num_mb);
2800                 test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2801                                        speed_template_16_32, num_mb);
2802                 test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2803                                        speed_template_16_32, num_mb);
2804                 test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2805                                        speed_template_16_32, num_mb);
2806                 test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2807                                        speed_template_16_32, num_mb);
2808                 test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2809                                        speed_template_16_32, num_mb);
2810                 test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2811                                        speed_template_32_48, num_mb);
2812                 test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2813                                        speed_template_32_48, num_mb);
2814                 test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2815                                        speed_template_32_64, num_mb);
2816                 test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2817                                        speed_template_32_64, num_mb);
2818                 break;
2819
2820         case 608:
2821                 test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2822                                        speed_template_16_32, num_mb);
2823                 test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2824                                        speed_template_16_32, num_mb);
2825                 test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2826                                        speed_template_16_32, num_mb);
2827                 test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2828                                        speed_template_16_32, num_mb);
2829                 test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2830                                        speed_template_16_32, num_mb);
2831                 test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2832                                        speed_template_16_32, num_mb);
2833                 test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2834                                        speed_template_32_48, num_mb);
2835                 test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2836                                        speed_template_32_48, num_mb);
2837                 test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2838                                        speed_template_32_64, num_mb);
2839                 test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2840                                        speed_template_32_64, num_mb);
2841                 break;
2842
2843         case 609:
2844                 test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2845                                        speed_template_8_32, num_mb);
2846                 test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2847                                        speed_template_8_32, num_mb);
2848                 test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2849                                        speed_template_8_32, num_mb);
2850                 test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2851                                        speed_template_8_32, num_mb);
2852                 test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2853                                        speed_template_8_32, num_mb);
2854                 test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2855                                        speed_template_8_32, num_mb);
2856                 break;
2857
2858         case 610:
2859                 test_mb_skcipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2860                                        speed_template_16_32, num_mb);
2861                 test_mb_skcipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2862                                        speed_template_16_32, num_mb);
2863                 test_mb_skcipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2864                                        speed_template_16_32, num_mb);
2865                 test_mb_skcipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2866                                        speed_template_16_32, num_mb);
2867                 break;
2868
2869         }
2870
2871         return ret;
2872 }
2873
2874 static int __init tcrypt_mod_init(void)
2875 {
2876         int err = -ENOMEM;
2877         int i;
2878
2879         for (i = 0; i < TVMEMSIZE; i++) {
2880                 tvmem[i] = (void *)__get_free_page(GFP_KERNEL);
2881                 if (!tvmem[i])
2882                         goto err_free_tv;
2883         }
2884
2885         err = do_test(alg, type, mask, mode, num_mb);
2886
2887         if (err) {
2888                 printk(KERN_ERR "tcrypt: one or more tests failed!\n");
2889                 goto err_free_tv;
2890         } else {
2891                 pr_debug("all tests passed\n");
2892         }
2893
2894         /* We intentionaly return -EAGAIN to prevent keeping the module,
2895          * unless we're running in fips mode. It does all its work from
2896          * init() and doesn't offer any runtime functionality, but in
2897          * the fips case, checking for a successful load is helpful.
2898          * => we don't need it in the memory, do we?
2899          *                                        -- mludvig
2900          */
2901         if (!fips_enabled)
2902                 err = -EAGAIN;
2903
2904 err_free_tv:
2905         for (i = 0; i < TVMEMSIZE && tvmem[i]; i++)
2906                 free_page((unsigned long)tvmem[i]);
2907
2908         return err;
2909 }
2910
2911 /*
2912  * If an init function is provided, an exit function must also be provided
2913  * to allow module unload.
2914  */
2915 static void __exit tcrypt_mod_fini(void) { }
2916
2917 late_initcall(tcrypt_mod_init);
2918 module_exit(tcrypt_mod_fini);
2919
2920 module_param(alg, charp, 0);
2921 module_param(type, uint, 0);
2922 module_param(mask, uint, 0);
2923 module_param(mode, int, 0);
2924 module_param(sec, uint, 0);
2925 MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
2926                       "(defaults to zero which uses CPU cycles instead)");
2927 module_param(num_mb, uint, 0000);
2928 MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
2929 module_param(klen, uint, 0);
2930 MODULE_PARM_DESC(klen, "Key length (defaults to 0)");
2931
2932 MODULE_LICENSE("GPL");
2933 MODULE_DESCRIPTION("Quick & dirty crypto testing module");
2934 MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>");