crypto/algapi.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Cryptographic API for algorithms (i.e., low-level API).
   4  *
   5  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   6  */
   7
   8 #include <crypto/algapi.h>
   9 #include <crypto/internal/simd.h>
  10 #include <linux/err.h>
  11 #include <linux/errno.h>
  12 #include <linux/fips.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel.h>
  15 #include <linux/list.h>
  16 #include <linux/module.h>
  17 #include <linux/rtnetlink.h>
  18 #include <linux/slab.h>
  19 #include <linux/string.h>
  20
  21 #include "internal.h"
  22
  23 static LIST_HEAD(crypto_template_list);
  24
  25 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
  26 DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
  27 EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
  28 #endif
  29
  30 static inline void crypto_check_module_sig(struct module *mod)
  31 {
  32         if (fips_enabled && mod && !module_sig_ok(mod))
  33                 panic("Module %s signature verification failed in FIPS mode\n",
  34                       module_name(mod));
  35 }
  36
  37 static int crypto_check_alg(struct crypto_alg *alg)
  38 {
  39         crypto_check_module_sig(alg->cra_module);
  40
  41         if (!alg->cra_name[0] || !alg->cra_driver_name[0])
  42                 return -EINVAL;
  43
  44         if (alg->cra_alignmask & (alg->cra_alignmask + 1))
  45                 return -EINVAL;
  46
  47         /* General maximums for all algs. */
  48         if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
  49                 return -EINVAL;
  50
  51         if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
  52                 return -EINVAL;
  53
  54         /* Lower maximums for specific alg types. */
  55         if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
  56                                CRYPTO_ALG_TYPE_CIPHER) {
  57                 if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
  58                         return -EINVAL;
  59
  60                 if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
  61                         return -EINVAL;
  62         }
  63
  64         if (alg->cra_priority < 0)
  65                 return -EINVAL;
  66
  67         refcount_set(&alg->cra_refcnt, 1);
  68
  69         return 0;
  70 }
  71
  72 static void crypto_free_instance(struct crypto_instance *inst)
  73 {
  74         inst->alg.cra_type->free(inst);
  75 }
  76
  77 static void crypto_destroy_instance(struct crypto_alg *alg)
  78 {
  79         struct crypto_instance *inst = (void *)alg;
  80         struct crypto_template *tmpl = inst->tmpl;
  81
  82         crypto_free_instance(inst);
  83         crypto_tmpl_put(tmpl);
  84 }
  85
  86 /*
  87  * This function adds a spawn to the list secondary_spawns which
  88  * will be used at the end of crypto_remove_spawns to unregister
  89  * instances, unless the spawn happens to be one that is depended
  90  * on by the new algorithm (nalg in crypto_remove_spawns).
  91  *
  92  * This function is also responsible for resurrecting any algorithms
  93  * in the dependency chain of nalg by unsetting n->dead.
  94  */
  95 static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
  96                                             struct list_head *stack,
  97                                             struct list_head *top,
  98                                             struct list_head *secondary_spawns)
  99 {
 100         struct crypto_spawn *spawn, *n;
 101
 102         spawn = list_first_entry_or_null(stack, struct crypto_spawn, list);
 103         if (!spawn)
 104                 return NULL;
 105
 106         n = list_prev_entry(spawn, list);
 107         list_move(&spawn->list, secondary_spawns);
 108
 109         if (list_is_last(&n->list, stack))
 110                 return top;
 111
 112         n = list_next_entry(n, list);
 113         if (!spawn->dead)
 114                 n->dead = false;
 115
 116         return &n->inst->alg.cra_users;
 117 }
 118
 119 static void crypto_remove_instance(struct crypto_instance *inst,
 120                                    struct list_head *list)
 121 {
 122         struct crypto_template *tmpl = inst->tmpl;
 123
 124         if (crypto_is_dead(&inst->alg))
 125                 return;
 126
 127         inst->alg.cra_flags |= CRYPTO_ALG_DEAD;
 128
 129         if (!tmpl || !crypto_tmpl_get(tmpl))
 130                 return;
 131
 132         list_move(&inst->alg.cra_list, list);
 133         hlist_del(&inst->list);
 134         inst->alg.cra_destroy = crypto_destroy_instance;
 135
 136         BUG_ON(!list_empty(&inst->alg.cra_users));
 137 }
 138
 139 /*
 140  * Given an algorithm alg, remove all algorithms that depend on it
 141  * through spawns.  If nalg is not null, then exempt any algorithms
 142  * that is depended on by nalg.  This is useful when nalg itself
 143  * depends on alg.
 144  */
 145 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
 146                           struct crypto_alg *nalg)
 147 {
 148         u32 new_type = (nalg ?: alg)->cra_flags;
 149         struct crypto_spawn *spawn, *n;
 150         LIST_HEAD(secondary_spawns);
 151         struct list_head *spawns;
 152         LIST_HEAD(stack);
 153         LIST_HEAD(top);
 154
 155         spawns = &alg->cra_users;
 156         list_for_each_entry_safe(spawn, n, spawns, list) {
 157                 if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
 158                         continue;
 159
 160                 list_move(&spawn->list, &top);
 161         }
 162
 163         /*
 164          * Perform a depth-first walk starting from alg through
 165          * the cra_users tree.  The list stack records the path
 166          * from alg to the current spawn.
 167          */
 168         spawns = &top;
 169         do {
 170                 while (!list_empty(spawns)) {
 171                         struct crypto_instance *inst;
 172
 173                         spawn = list_first_entry(spawns, struct crypto_spawn,
 174                                                  list);
 175                         inst = spawn->inst;
 176
 177                         list_move(&spawn->list, &stack);
 178                         spawn->dead = !spawn->registered || &inst->alg != nalg;
 179
 180                         if (!spawn->registered)
 181                                 break;
 182
 183                         BUG_ON(&inst->alg == alg);
 184
 185                         if (&inst->alg == nalg)
 186                                 break;
 187
 188                         spawns = &inst->alg.cra_users;
 189
 190                         /*
 191                          * Even if spawn->registered is true, the
 192                          * instance itself may still be unregistered.
 193                          * This is because it may have failed during
 194                          * registration.  Therefore we still need to
 195                          * make the following test.
 196                          *
 197                          * We may encounter an unregistered instance here, since
 198                          * an instance's spawns are set up prior to the instance
 199                          * being registered.  An unregistered instance will have
 200                          * NULL ->cra_users.next, since ->cra_users isn't
 201                          * properly initialized until registration.  But an
 202                          * unregistered instance cannot have any users, so treat
 203                          * it the same as ->cra_users being empty.
 204                          */
 205                         if (spawns->next == NULL)
 206                                 break;
 207                 }
 208         } while ((spawns = crypto_more_spawns(alg, &stack, &top,
 209                                               &secondary_spawns)));
 210
 211         /*
 212          * Remove all instances that are marked as dead.  Also
 213          * complete the resurrection of the others by moving them
 214          * back to the cra_users list.
 215          */
 216         list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
 217                 if (!spawn->dead)
 218                         list_move(&spawn->list, &spawn->alg->cra_users);
 219                 else if (spawn->registered)
 220                         crypto_remove_instance(spawn->inst, list);
 221         }
 222 }
 223 EXPORT_SYMBOL_GPL(crypto_remove_spawns);
 224
 225 static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)
 226 {
 227         struct crypto_larval *larval;
 228
 229         if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER))
 230                 return NULL;
 231
 232         larval = crypto_larval_alloc(alg->cra_name,
 233                                      alg->cra_flags | CRYPTO_ALG_TESTED, 0);
 234         if (IS_ERR(larval))
 235                 return larval;
 236
 237         larval->adult = crypto_mod_get(alg);
 238         if (!larval->adult) {
 239                 kfree(larval);
 240                 return ERR_PTR(-ENOENT);
 241         }
 242
 243         refcount_set(&larval->alg.cra_refcnt, 1);
 244         memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
 245                CRYPTO_MAX_ALG_NAME);
 246         larval->alg.cra_priority = alg->cra_priority;
 247
 248         return larval;
 249 }
 250
 251 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)
 252 {
 253         struct crypto_alg *q;
 254         struct crypto_larval *larval;
 255         int ret = -EAGAIN;
 256
 257         if (crypto_is_dead(alg))
 258                 goto err;
 259
 260         INIT_LIST_HEAD(&alg->cra_users);
 261
 262         /* No cheating! */
 263         alg->cra_flags &= ~CRYPTO_ALG_TESTED;
 264
 265         ret = -EEXIST;
 266
 267         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 268                 if (q == alg)
 269                         goto err;
 270
 271                 if (crypto_is_moribund(q))
 272                         continue;
 273
 274                 if (crypto_is_larval(q)) {
 275                         if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
 276                                 goto err;
 277                         continue;
 278                 }
 279
 280                 if (!strcmp(q->cra_driver_name, alg->cra_name) ||
 281                     !strcmp(q->cra_name, alg->cra_driver_name))
 282                         goto err;
 283         }
 284
 285         larval = crypto_alloc_test_larval(alg);
 286         if (IS_ERR(larval))
 287                 goto out;
 288
 289         list_add(&alg->cra_list, &crypto_alg_list);
 290
 291         if (larval)
 292                 list_add(&larval->alg.cra_list, &crypto_alg_list);
 293         else
 294                 alg->cra_flags |= CRYPTO_ALG_TESTED;
 295
 296         crypto_stats_init(alg);
 297
 298 out:
 299         return larval;
 300
 301 err:
 302         larval = ERR_PTR(ret);
 303         goto out;
 304 }
 305
 306 void crypto_alg_tested(const char *name, int err)
 307 {
 308         struct crypto_larval *test;
 309         struct crypto_alg *alg;
 310         struct crypto_alg *q;
 311         LIST_HEAD(list);
 312         bool best;
 313
 314         down_write(&crypto_alg_sem);
 315         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 316                 if (crypto_is_moribund(q) || !crypto_is_larval(q))
 317                         continue;
 318
 319                 test = (struct crypto_larval *)q;
 320
 321                 if (!strcmp(q->cra_driver_name, name))
 322                         goto found;
 323         }
 324
 325         pr_err("alg: Unexpected test result for %s: %d\n", name, err);
 326         goto unlock;
 327
 328 found:
 329         q->cra_flags |= CRYPTO_ALG_DEAD;
 330         alg = test->adult;
 331
 332         if (list_empty(&alg->cra_list))
 333                 goto complete;
 334
 335         if (err == -ECANCELED)
 336                 alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;
 337         else if (err)
 338                 goto complete;
 339         else
 340                 alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;
 341
 342         alg->cra_flags |= CRYPTO_ALG_TESTED;
 343
 344         /* Only satisfy larval waiters if we are the best. */
 345         best = true;
 346         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 347                 if (crypto_is_moribund(q) || !crypto_is_larval(q))
 348                         continue;
 349
 350                 if (strcmp(alg->cra_name, q->cra_name))
 351                         continue;
 352
 353                 if (q->cra_priority > alg->cra_priority) {
 354                         best = false;
 355                         break;
 356                 }
 357         }
 358
 359         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 360                 if (q == alg)
 361                         continue;
 362
 363                 if (crypto_is_moribund(q))
 364                         continue;
 365
 366                 if (crypto_is_larval(q)) {
 367                         struct crypto_larval *larval = (void *)q;
 368
 369                         /*
 370                          * Check to see if either our generic name or
 371                          * specific name can satisfy the name requested
 372                          * by the larval entry q.
 373                          */
 374                         if (strcmp(alg->cra_name, q->cra_name) &&
 375                             strcmp(alg->cra_driver_name, q->cra_name))
 376                                 continue;
 377
 378                         if (larval->adult)
 379                                 continue;
 380                         if ((q->cra_flags ^ alg->cra_flags) & larval->mask)
 381                                 continue;
 382
 383                         if (best && crypto_mod_get(alg))
 384                                 larval->adult = alg;
 385                         else
 386                                 larval->adult = ERR_PTR(-EAGAIN);
 387
 388                         continue;
 389                 }
 390
 391                 if (strcmp(alg->cra_name, q->cra_name))
 392                         continue;
 393
 394                 if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
 395                     q->cra_priority > alg->cra_priority)
 396                         continue;
 397
 398                 crypto_remove_spawns(q, &list, alg);
 399         }
 400
 401 complete:
 402         complete_all(&test->completion);
 403
 404 unlock:
 405         up_write(&crypto_alg_sem);
 406
 407         crypto_remove_final(&list);
 408 }
 409 EXPORT_SYMBOL_GPL(crypto_alg_tested);
 410
 411 void crypto_remove_final(struct list_head *list)
 412 {
 413         struct crypto_alg *alg;
 414         struct crypto_alg *n;
 415
 416         list_for_each_entry_safe(alg, n, list, cra_list) {
 417                 list_del_init(&alg->cra_list);
 418                 crypto_alg_put(alg);
 419         }
 420 }
 421 EXPORT_SYMBOL_GPL(crypto_remove_final);
 422
 423 int crypto_register_alg(struct crypto_alg *alg)
 424 {
 425         struct crypto_larval *larval;
 426         bool test_started;
 427         int err;
 428
 429         alg->cra_flags &= ~CRYPTO_ALG_DEAD;
 430         err = crypto_check_alg(alg);
 431         if (err)
 432                 return err;
 433
 434         down_write(&crypto_alg_sem);
 435         larval = __crypto_register_alg(alg);
 436         test_started = static_key_enabled(&crypto_boot_test_finished);
 437         if (!IS_ERR_OR_NULL(larval))
 438                 larval->test_started = test_started;
 439         up_write(&crypto_alg_sem);
 440
 441         if (IS_ERR_OR_NULL(larval))
 442                 return PTR_ERR(larval);
 443
 444         if (test_started)
 445                 crypto_wait_for_test(larval);
 446         return 0;
 447 }
 448 EXPORT_SYMBOL_GPL(crypto_register_alg);
 449
 450 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
 451 {
 452         if (unlikely(list_empty(&alg->cra_list)))
 453                 return -ENOENT;
 454
 455         alg->cra_flags |= CRYPTO_ALG_DEAD;
 456
 457         list_del_init(&alg->cra_list);
 458         crypto_remove_spawns(alg, list, NULL);
 459
 460         return 0;
 461 }
 462
 463 void crypto_unregister_alg(struct crypto_alg *alg)
 464 {
 465         int ret;
 466         LIST_HEAD(list);
 467
 468         down_write(&crypto_alg_sem);
 469         ret = crypto_remove_alg(alg, &list);
 470         up_write(&crypto_alg_sem);
 471
 472         if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
 473                 return;
 474
 475         BUG_ON(refcount_read(&alg->cra_refcnt) != 1);
 476         if (alg->cra_destroy)
 477                 alg->cra_destroy(alg);
 478
 479         crypto_remove_final(&list);
 480 }
 481 EXPORT_SYMBOL_GPL(crypto_unregister_alg);
 482
 483 int crypto_register_algs(struct crypto_alg *algs, int count)
 484 {
 485         int i, ret;
 486
 487         for (i = 0; i < count; i++) {
 488                 ret = crypto_register_alg(&algs[i]);
 489                 if (ret)
 490                         goto err;
 491         }
 492
 493         return 0;
 494
 495 err:
 496         for (--i; i >= 0; --i)
 497                 crypto_unregister_alg(&algs[i]);
 498
 499         return ret;
 500 }
 501 EXPORT_SYMBOL_GPL(crypto_register_algs);
 502
 503 void crypto_unregister_algs(struct crypto_alg *algs, int count)
 504 {
 505         int i;
 506
 507         for (i = 0; i < count; i++)
 508                 crypto_unregister_alg(&algs[i]);
 509 }
 510 EXPORT_SYMBOL_GPL(crypto_unregister_algs);
 511
 512 int crypto_register_template(struct crypto_template *tmpl)
 513 {
 514         struct crypto_template *q;
 515         int err = -EEXIST;
 516
 517         down_write(&crypto_alg_sem);
 518
 519         crypto_check_module_sig(tmpl->module);
 520
 521         list_for_each_entry(q, &crypto_template_list, list) {
 522                 if (q == tmpl)
 523                         goto out;
 524         }
 525
 526         list_add(&tmpl->list, &crypto_template_list);
 527         err = 0;
 528 out:
 529         up_write(&crypto_alg_sem);
 530         return err;
 531 }
 532 EXPORT_SYMBOL_GPL(crypto_register_template);
 533
 534 int crypto_register_templates(struct crypto_template *tmpls, int count)
 535 {
 536         int i, err;
 537
 538         for (i = 0; i < count; i++) {
 539                 err = crypto_register_template(&tmpls[i]);
 540                 if (err)
 541                         goto out;
 542         }
 543         return 0;
 544
 545 out:
 546         for (--i; i >= 0; --i)
 547                 crypto_unregister_template(&tmpls[i]);
 548         return err;
 549 }
 550 EXPORT_SYMBOL_GPL(crypto_register_templates);
 551
 552 void crypto_unregister_template(struct crypto_template *tmpl)
 553 {
 554         struct crypto_instance *inst;
 555         struct hlist_node *n;
 556         struct hlist_head *list;
 557         LIST_HEAD(users);
 558
 559         down_write(&crypto_alg_sem);
 560
 561         BUG_ON(list_empty(&tmpl->list));
 562         list_del_init(&tmpl->list);
 563
 564         list = &tmpl->instances;
 565         hlist_for_each_entry(inst, list, list) {
 566                 int err = crypto_remove_alg(&inst->alg, &users);
 567
 568                 BUG_ON(err);
 569         }
 570
 571         up_write(&crypto_alg_sem);
 572
 573         hlist_for_each_entry_safe(inst, n, list, list) {
 574                 BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
 575                 crypto_free_instance(inst);
 576         }
 577         crypto_remove_final(&users);
 578 }
 579 EXPORT_SYMBOL_GPL(crypto_unregister_template);
 580
 581 void crypto_unregister_templates(struct crypto_template *tmpls, int count)
 582 {
 583         int i;
 584
 585         for (i = count - 1; i >= 0; --i)
 586                 crypto_unregister_template(&tmpls[i]);
 587 }
 588 EXPORT_SYMBOL_GPL(crypto_unregister_templates);
 589
 590 static struct crypto_template *__crypto_lookup_template(const char *name)
 591 {
 592         struct crypto_template *q, *tmpl = NULL;
 593
 594         down_read(&crypto_alg_sem);
 595         list_for_each_entry(q, &crypto_template_list, list) {
 596                 if (strcmp(q->name, name))
 597                         continue;
 598                 if (unlikely(!crypto_tmpl_get(q)))
 599                         continue;
 600
 601                 tmpl = q;
 602                 break;
 603         }
 604         up_read(&crypto_alg_sem);
 605
 606         return tmpl;
 607 }
 608
 609 struct crypto_template *crypto_lookup_template(const char *name)
 610 {
 611         return try_then_request_module(__crypto_lookup_template(name),
 612                                        "crypto-%s", name);
 613 }
 614 EXPORT_SYMBOL_GPL(crypto_lookup_template);
 615
 616 int crypto_register_instance(struct crypto_template *tmpl,
 617                              struct crypto_instance *inst)
 618 {
 619         struct crypto_larval *larval;
 620         struct crypto_spawn *spawn;
 621         u32 fips_internal = 0;
 622         int err;
 623
 624         err = crypto_check_alg(&inst->alg);
 625         if (err)
 626                 return err;
 627
 628         inst->alg.cra_module = tmpl->module;
 629         inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
 630
 631         down_write(&crypto_alg_sem);
 632
 633         larval = ERR_PTR(-EAGAIN);
 634         for (spawn = inst->spawns; spawn;) {
 635                 struct crypto_spawn *next;
 636
 637                 if (spawn->dead)
 638                         goto unlock;
 639
 640                 next = spawn->next;
 641                 spawn->inst = inst;
 642                 spawn->registered = true;
 643
 644                 fips_internal |= spawn->alg->cra_flags;
 645
 646                 crypto_mod_put(spawn->alg);
 647
 648                 spawn = next;
 649         }
 650
 651         inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);
 652
 653         larval = __crypto_register_alg(&inst->alg);
 654         if (IS_ERR(larval))
 655                 goto unlock;
 656         else if (larval)
 657                 larval->test_started = true;
 658
 659         hlist_add_head(&inst->list, &tmpl->instances);
 660         inst->tmpl = tmpl;
 661
 662 unlock:
 663         up_write(&crypto_alg_sem);
 664
 665         err = PTR_ERR(larval);
 666         if (IS_ERR_OR_NULL(larval))
 667                 goto err;
 668
 669         crypto_wait_for_test(larval);
 670         err = 0;
 671
 672 err:
 673         return err;
 674 }
 675 EXPORT_SYMBOL_GPL(crypto_register_instance);
 676
 677 void crypto_unregister_instance(struct crypto_instance *inst)
 678 {
 679         LIST_HEAD(list);
 680
 681         down_write(&crypto_alg_sem);
 682
 683         crypto_remove_spawns(&inst->alg, &list, NULL);
 684         crypto_remove_instance(inst, &list);
 685
 686         up_write(&crypto_alg_sem);
 687
 688         crypto_remove_final(&list);
 689 }
 690 EXPORT_SYMBOL_GPL(crypto_unregister_instance);
 691
 692 int crypto_grab_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst,
 693                       const char *name, u32 type, u32 mask)
 694 {
 695         struct crypto_alg *alg;
 696         int err = -EAGAIN;
 697
 698         if (WARN_ON_ONCE(inst == NULL))
 699                 return -EINVAL;
 700
 701         /* Allow the result of crypto_attr_alg_name() to be passed directly */
 702         if (IS_ERR(name))
 703                 return PTR_ERR(name);
 704
 705         alg = crypto_find_alg(name, spawn->frontend,
 706                               type | CRYPTO_ALG_FIPS_INTERNAL, mask);
 707         if (IS_ERR(alg))
 708                 return PTR_ERR(alg);
 709
 710         down_write(&crypto_alg_sem);
 711         if (!crypto_is_moribund(alg)) {
 712                 list_add(&spawn->list, &alg->cra_users);
 713                 spawn->alg = alg;
 714                 spawn->mask = mask;
 715                 spawn->next = inst->spawns;
 716                 inst->spawns = spawn;
 717                 inst->alg.cra_flags |=
 718                         (alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
 719                 err = 0;
 720         }
 721         up_write(&crypto_alg_sem);
 722         if (err)
 723                 crypto_mod_put(alg);
 724         return err;
 725 }
 726 EXPORT_SYMBOL_GPL(crypto_grab_spawn);
 727
 728 void crypto_drop_spawn(struct crypto_spawn *spawn)
 729 {
 730         if (!spawn->alg) /* not yet initialized? */
 731                 return;
 732
 733         down_write(&crypto_alg_sem);
 734         if (!spawn->dead)
 735                 list_del(&spawn->list);
 736         up_write(&crypto_alg_sem);
 737
 738         if (!spawn->registered)
 739                 crypto_mod_put(spawn->alg);
 740 }
 741 EXPORT_SYMBOL_GPL(crypto_drop_spawn);
 742
 743 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn)
 744 {
 745         struct crypto_alg *alg = ERR_PTR(-EAGAIN);
 746         struct crypto_alg *target;
 747         bool shoot = false;
 748
 749         down_read(&crypto_alg_sem);
 750         if (!spawn->dead) {
 751                 alg = spawn->alg;
 752                 if (!crypto_mod_get(alg)) {
 753                         target = crypto_alg_get(alg);
 754                         shoot = true;
 755                         alg = ERR_PTR(-EAGAIN);
 756                 }
 757         }
 758         up_read(&crypto_alg_sem);
 759
 760         if (shoot) {
 761                 crypto_shoot_alg(target);
 762                 crypto_alg_put(target);
 763         }
 764
 765         return alg;
 766 }
 767
 768 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type,
 769                                     u32 mask)
 770 {
 771         struct crypto_alg *alg;
 772         struct crypto_tfm *tfm;
 773
 774         alg = crypto_spawn_alg(spawn);
 775         if (IS_ERR(alg))
 776                 return ERR_CAST(alg);
 777
 778         tfm = ERR_PTR(-EINVAL);
 779         if (unlikely((alg->cra_flags ^ type) & mask))
 780                 goto out_put_alg;
 781
 782         tfm = __crypto_alloc_tfm(alg, type, mask);
 783         if (IS_ERR(tfm))
 784                 goto out_put_alg;
 785
 786         return tfm;
 787
 788 out_put_alg:
 789         crypto_mod_put(alg);
 790         return tfm;
 791 }
 792 EXPORT_SYMBOL_GPL(crypto_spawn_tfm);
 793
 794 void *crypto_spawn_tfm2(struct crypto_spawn *spawn)
 795 {
 796         struct crypto_alg *alg;
 797         struct crypto_tfm *tfm;
 798
 799         alg = crypto_spawn_alg(spawn);
 800         if (IS_ERR(alg))
 801                 return ERR_CAST(alg);
 802
 803         tfm = crypto_create_tfm(alg, spawn->frontend);
 804         if (IS_ERR(tfm))
 805                 goto out_put_alg;
 806
 807         return tfm;
 808
 809 out_put_alg:
 810         crypto_mod_put(alg);
 811         return tfm;
 812 }
 813 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);
 814
 815 int crypto_register_notifier(struct notifier_block *nb)
 816 {
 817         return blocking_notifier_chain_register(&crypto_chain, nb);
 818 }
 819 EXPORT_SYMBOL_GPL(crypto_register_notifier);
 820
 821 int crypto_unregister_notifier(struct notifier_block *nb)
 822 {
 823         return blocking_notifier_chain_unregister(&crypto_chain, nb);
 824 }
 825 EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
 826
 827 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
 828 {
 829         struct rtattr *rta = tb[0];
 830         struct crypto_attr_type *algt;
 831
 832         if (!rta)
 833                 return ERR_PTR(-ENOENT);
 834         if (RTA_PAYLOAD(rta) < sizeof(*algt))
 835                 return ERR_PTR(-EINVAL);
 836         if (rta->rta_type != CRYPTOA_TYPE)
 837                 return ERR_PTR(-EINVAL);
 838
 839         algt = RTA_DATA(rta);
 840
 841         return algt;
 842 }
 843 EXPORT_SYMBOL_GPL(crypto_get_attr_type);
 844
 845 /**
 846  * crypto_check_attr_type() - check algorithm type and compute inherited mask
 847  * @tb: the template parameters
 848  * @type: the algorithm type the template would be instantiated as
 849  * @mask_ret: (output) the mask that should be passed to crypto_grab_*()
 850  *            to restrict the flags of any inner algorithms
 851  *
 852  * Validate that the algorithm type the user requested is compatible with the
 853  * one the template would actually be instantiated as.  E.g., if the user is
 854  * doing crypto_alloc_shash("cbc(aes)", ...), this would return an error because
 855  * the "cbc" template creates an "skcipher" algorithm, not an "shash" algorithm.
 856  *
 857  * Also compute the mask to use to restrict the flags of any inner algorithms.
 858  *
 859  * Return: 0 on success; -errno on failure
 860  */
 861 int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret)
 862 {
 863         struct crypto_attr_type *algt;
 864
 865         algt = crypto_get_attr_type(tb);
 866         if (IS_ERR(algt))
 867                 return PTR_ERR(algt);
 868
 869         if ((algt->type ^ type) & algt->mask)
 870                 return -EINVAL;
 871
 872         *mask_ret = crypto_algt_inherited_mask(algt);
 873         return 0;
 874 }
 875 EXPORT_SYMBOL_GPL(crypto_check_attr_type);
 876
 877 const char *crypto_attr_alg_name(struct rtattr *rta)
 878 {
 879         struct crypto_attr_alg *alga;
 880
 881         if (!rta)
 882                 return ERR_PTR(-ENOENT);
 883         if (RTA_PAYLOAD(rta) < sizeof(*alga))
 884                 return ERR_PTR(-EINVAL);
 885         if (rta->rta_type != CRYPTOA_ALG)
 886                 return ERR_PTR(-EINVAL);
 887
 888         alga = RTA_DATA(rta);
 889         alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
 890
 891         return alga->name;
 892 }
 893 EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
 894
 895 int crypto_inst_setname(struct crypto_instance *inst, const char *name,
 896                         struct crypto_alg *alg)
 897 {
 898         if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
 899                      alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
 900                 return -ENAMETOOLONG;
 901
 902         if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
 903                      name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 904                 return -ENAMETOOLONG;
 905
 906         return 0;
 907 }
 908 EXPORT_SYMBOL_GPL(crypto_inst_setname);
 909
 910 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
 911 {
 912         INIT_LIST_HEAD(&queue->list);
 913         queue->backlog = &queue->list;
 914         queue->qlen = 0;
 915         queue->max_qlen = max_qlen;
 916 }
 917 EXPORT_SYMBOL_GPL(crypto_init_queue);
 918
 919 int crypto_enqueue_request(struct crypto_queue *queue,
 920                            struct crypto_async_request *request)
 921 {
 922         int err = -EINPROGRESS;
 923
 924         if (unlikely(queue->qlen >= queue->max_qlen)) {
 925                 if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 926                         err = -ENOSPC;
 927                         goto out;
 928                 }
 929                 err = -EBUSY;
 930                 if (queue->backlog == &queue->list)
 931                         queue->backlog = &request->list;
 932         }
 933
 934         queue->qlen++;
 935         list_add_tail(&request->list, &queue->list);
 936
 937 out:
 938         return err;
 939 }
 940 EXPORT_SYMBOL_GPL(crypto_enqueue_request);
 941
 942 void crypto_enqueue_request_head(struct crypto_queue *queue,
 943                                  struct crypto_async_request *request)
 944 {
 945         queue->qlen++;
 946         list_add(&request->list, &queue->list);
 947 }
 948 EXPORT_SYMBOL_GPL(crypto_enqueue_request_head);
 949
 950 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
 951 {
 952         struct list_head *request;
 953
 954         if (unlikely(!queue->qlen))
 955                 return NULL;
 956
 957         queue->qlen--;
 958
 959         if (queue->backlog != &queue->list)
 960                 queue->backlog = queue->backlog->next;
 961
 962         request = queue->list.next;
 963         list_del(request);
 964
 965         return list_entry(request, struct crypto_async_request, list);
 966 }
 967 EXPORT_SYMBOL_GPL(crypto_dequeue_request);
 968
 969 static inline void crypto_inc_byte(u8 *a, unsigned int size)
 970 {
 971         u8 *b = (a + size);
 972         u8 c;
 973
 974         for (; size; size--) {
 975                 c = *--b + 1;
 976                 *b = c;
 977                 if (c)
 978                         break;
 979         }
 980 }
 981
 982 void crypto_inc(u8 *a, unsigned int size)
 983 {
 984         __be32 *b = (__be32 *)(a + size);
 985         u32 c;
 986
 987         if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
 988             IS_ALIGNED((unsigned long)b, __alignof__(*b)))
 989                 for (; size >= 4; size -= 4) {
 990                         c = be32_to_cpu(*--b) + 1;
 991                         *b = cpu_to_be32(c);
 992                         if (likely(c))
 993                                 return;
 994                 }
 995
 996         crypto_inc_byte(a, size);
 997 }
 998 EXPORT_SYMBOL_GPL(crypto_inc);
 999
1000 unsigned int crypto_alg_extsize(struct crypto_alg *alg)
1001 {
1002         return alg->cra_ctxsize +
1003                (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
1004 }
1005 EXPORT_SYMBOL_GPL(crypto_alg_extsize);
1006
1007 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
1008                         u32 type, u32 mask)
1009 {
1010         int ret = 0;
1011         struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);
1012
1013         if (!IS_ERR(alg)) {
1014                 crypto_mod_put(alg);
1015                 ret = 1;
1016         }
1017
1018         return ret;
1019 }
1020 EXPORT_SYMBOL_GPL(crypto_type_has_alg);
1021
1022 #ifdef CONFIG_CRYPTO_STATS
1023 void crypto_stats_init(struct crypto_alg *alg)
1024 {
1025         memset(&alg->stats, 0, sizeof(alg->stats));
1026 }
1027 EXPORT_SYMBOL_GPL(crypto_stats_init);
1028
1029 void crypto_stats_get(struct crypto_alg *alg)
1030 {
1031         crypto_alg_get(alg);
1032 }
1033 EXPORT_SYMBOL_GPL(crypto_stats_get);
1034
1035 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg,
1036                                int ret)
1037 {
1038         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1039                 atomic64_inc(&alg->stats.aead.err_cnt);
1040         } else {
1041                 atomic64_inc(&alg->stats.aead.encrypt_cnt);
1042                 atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen);
1043         }
1044         crypto_alg_put(alg);
1045 }
1046 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt);
1047
1048 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg,
1049                                int ret)
1050 {
1051         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1052                 atomic64_inc(&alg->stats.aead.err_cnt);
1053         } else {
1054                 atomic64_inc(&alg->stats.aead.decrypt_cnt);
1055                 atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen);
1056         }
1057         crypto_alg_put(alg);
1058 }
1059 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt);
1060
1061 void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret,
1062                                    struct crypto_alg *alg)
1063 {
1064         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1065                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1066         } else {
1067                 atomic64_inc(&alg->stats.akcipher.encrypt_cnt);
1068                 atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen);
1069         }
1070         crypto_alg_put(alg);
1071 }
1072 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt);
1073
1074 void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret,
1075                                    struct crypto_alg *alg)
1076 {
1077         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1078                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1079         } else {
1080                 atomic64_inc(&alg->stats.akcipher.decrypt_cnt);
1081                 atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen);
1082         }
1083         crypto_alg_put(alg);
1084 }
1085 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt);
1086
1087 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg)
1088 {
1089         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1090                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1091         else
1092                 atomic64_inc(&alg->stats.akcipher.sign_cnt);
1093         crypto_alg_put(alg);
1094 }
1095 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign);
1096
1097 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg)
1098 {
1099         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1100                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1101         else
1102                 atomic64_inc(&alg->stats.akcipher.verify_cnt);
1103         crypto_alg_put(alg);
1104 }
1105 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify);
1106
1107 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg)
1108 {
1109         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1110                 atomic64_inc(&alg->stats.compress.err_cnt);
1111         } else {
1112                 atomic64_inc(&alg->stats.compress.compress_cnt);
1113                 atomic64_add(slen, &alg->stats.compress.compress_tlen);
1114         }
1115         crypto_alg_put(alg);
1116 }
1117 EXPORT_SYMBOL_GPL(crypto_stats_compress);
1118
1119 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg)
1120 {
1121         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1122                 atomic64_inc(&alg->stats.compress.err_cnt);
1123         } else {
1124                 atomic64_inc(&alg->stats.compress.decompress_cnt);
1125                 atomic64_add(slen, &alg->stats.compress.decompress_tlen);
1126         }
1127         crypto_alg_put(alg);
1128 }
1129 EXPORT_SYMBOL_GPL(crypto_stats_decompress);
1130
1131 void crypto_stats_ahash_update(unsigned int nbytes, int ret,
1132                                struct crypto_alg *alg)
1133 {
1134         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1135                 atomic64_inc(&alg->stats.hash.err_cnt);
1136         else
1137                 atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
1138         crypto_alg_put(alg);
1139 }
1140 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update);
1141
1142 void crypto_stats_ahash_final(unsigned int nbytes, int ret,
1143                               struct crypto_alg *alg)
1144 {
1145         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1146                 atomic64_inc(&alg->stats.hash.err_cnt);
1147         } else {
1148                 atomic64_inc(&alg->stats.hash.hash_cnt);
1149                 atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
1150         }
1151         crypto_alg_put(alg);
1152 }
1153 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final);
1154
1155 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret)
1156 {
1157         if (ret)
1158                 atomic64_inc(&alg->stats.kpp.err_cnt);
1159         else
1160                 atomic64_inc(&alg->stats.kpp.setsecret_cnt);
1161         crypto_alg_put(alg);
1162 }
1163 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret);
1164
1165 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret)
1166 {
1167         if (ret)
1168                 atomic64_inc(&alg->stats.kpp.err_cnt);
1169         else
1170                 atomic64_inc(&alg->stats.kpp.generate_public_key_cnt);
1171         crypto_alg_put(alg);
1172 }
1173 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key);
1174
1175 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret)
1176 {
1177         if (ret)
1178                 atomic64_inc(&alg->stats.kpp.err_cnt);
1179         else
1180                 atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt);
1181         crypto_alg_put(alg);
1182 }
1183 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret);
1184
1185 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret)
1186 {
1187         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1188                 atomic64_inc(&alg->stats.rng.err_cnt);
1189         else
1190                 atomic64_inc(&alg->stats.rng.seed_cnt);
1191         crypto_alg_put(alg);
1192 }
1193 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed);
1194
1195 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen,
1196                                int ret)
1197 {
1198         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1199                 atomic64_inc(&alg->stats.rng.err_cnt);
1200         } else {
1201                 atomic64_inc(&alg->stats.rng.generate_cnt);
1202                 atomic64_add(dlen, &alg->stats.rng.generate_tlen);
1203         }
1204         crypto_alg_put(alg);
1205 }
1206 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate);
1207
1208 void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret,
1209                                    struct crypto_alg *alg)
1210 {
1211         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1212                 atomic64_inc(&alg->stats.cipher.err_cnt);
1213         } else {
1214                 atomic64_inc(&alg->stats.cipher.encrypt_cnt);
1215                 atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen);
1216         }
1217         crypto_alg_put(alg);
1218 }
1219 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt);
1220
1221 void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret,
1222                                    struct crypto_alg *alg)
1223 {
1224         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1225                 atomic64_inc(&alg->stats.cipher.err_cnt);
1226         } else {
1227                 atomic64_inc(&alg->stats.cipher.decrypt_cnt);
1228                 atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen);
1229         }
1230         crypto_alg_put(alg);
1231 }
1232 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt);
1233 #endif
1234
1235 static void __init crypto_start_tests(void)
1236 {
1237         for (;;) {
1238                 struct crypto_larval *larval = NULL;
1239                 struct crypto_alg *q;
1240
1241                 down_write(&crypto_alg_sem);
1242
1243                 list_for_each_entry(q, &crypto_alg_list, cra_list) {
1244                         struct crypto_larval *l;
1245
1246                         if (!crypto_is_larval(q))
1247                                 continue;
1248
1249                         l = (void *)q;
1250
1251                         if (!crypto_is_test_larval(l))
1252                                 continue;
1253
1254                         if (l->test_started)
1255                                 continue;
1256
1257                         l->test_started = true;
1258                         larval = l;
1259                         break;
1260                 }
1261
1262                 up_write(&crypto_alg_sem);
1263
1264                 if (!larval)
1265                         break;
1266
1267                 crypto_wait_for_test(larval);
1268         }
1269
1270         static_branch_enable(&crypto_boot_test_finished);
1271 }
1272
1273 static int __init crypto_algapi_init(void)
1274 {
1275         crypto_init_proc();
1276         crypto_start_tests();
1277         return 0;
1278 }
1279
1280 static void __exit crypto_algapi_exit(void)
1281 {
1282         crypto_exit_proc();
1283 }
1284
1285 /*
1286  * We run this at late_initcall so that all the built-in algorithms
1287  * have had a chance to register themselves first.
1288  */
1289 late_initcall(crypto_algapi_init);
1290 module_exit(crypto_algapi_exit);
1291
1292 MODULE_LICENSE("GPL");
1293 MODULE_DESCRIPTION("Cryptographic algorithms API");
1294 MODULE_SOFTDEP("pre: cryptomgr");