crypto/algapi.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Cryptographic API for algorithms (i.e., low-level API).
   4  *
   5  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   6  */
   7
   8 #include <crypto/algapi.h>
   9 #include <crypto/internal/simd.h>
  10 #include <linux/err.h>
  11 #include <linux/errno.h>
  12 #include <linux/fips.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel.h>
  15 #include <linux/list.h>
  16 #include <linux/module.h>
  17 #include <linux/rtnetlink.h>
  18 #include <linux/slab.h>
  19 #include <linux/string.h>
  20 #include <linux/workqueue.h>
  21
  22 #include "internal.h"
  23
  24 static LIST_HEAD(crypto_template_list);
  25
  26 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
  27 DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
  28 EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
  29 #endif
  30
  31 static inline void crypto_check_module_sig(struct module *mod)
  32 {
  33         if (fips_enabled && mod && !module_sig_ok(mod))
  34                 panic("Module %s signature verification failed in FIPS mode\n",
  35                       module_name(mod));
  36 }
  37
  38 static int crypto_check_alg(struct crypto_alg *alg)
  39 {
  40         crypto_check_module_sig(alg->cra_module);
  41
  42         if (!alg->cra_name[0] || !alg->cra_driver_name[0])
  43                 return -EINVAL;
  44
  45         if (alg->cra_alignmask & (alg->cra_alignmask + 1))
  46                 return -EINVAL;
  47
  48         /* General maximums for all algs. */
  49         if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
  50                 return -EINVAL;
  51
  52         if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
  53                 return -EINVAL;
  54
  55         /* Lower maximums for specific alg types. */
  56         if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
  57                                CRYPTO_ALG_TYPE_CIPHER) {
  58                 if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
  59                         return -EINVAL;
  60
  61                 if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
  62                         return -EINVAL;
  63         }
  64
  65         if (alg->cra_priority < 0)
  66                 return -EINVAL;
  67
  68         refcount_set(&alg->cra_refcnt, 1);
  69
  70         return 0;
  71 }
  72
  73 static void crypto_free_instance(struct crypto_instance *inst)
  74 {
  75         inst->alg.cra_type->free(inst);
  76 }
  77
  78 static void crypto_destroy_instance_workfn(struct work_struct *w)
  79 {
  80         struct crypto_instance *inst = container_of(w, struct crypto_instance,
  81                                                     free_work);
  82         struct crypto_template *tmpl = inst->tmpl;
  83
  84         crypto_free_instance(inst);
  85         crypto_tmpl_put(tmpl);
  86 }
  87
  88 static void crypto_destroy_instance(struct crypto_alg *alg)
  89 {
  90         struct crypto_instance *inst = container_of(alg,
  91                                                     struct crypto_instance,
  92                                                     alg);
  93
  94         INIT_WORK(&inst->free_work, crypto_destroy_instance_workfn);
  95         schedule_work(&inst->free_work);
  96 }
  97
  98 /*
  99  * This function adds a spawn to the list secondary_spawns which
 100  * will be used at the end of crypto_remove_spawns to unregister
 101  * instances, unless the spawn happens to be one that is depended
 102  * on by the new algorithm (nalg in crypto_remove_spawns).
 103  *
 104  * This function is also responsible for resurrecting any algorithms
 105  * in the dependency chain of nalg by unsetting n->dead.
 106  */
 107 static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
 108                                             struct list_head *stack,
 109                                             struct list_head *top,
 110                                             struct list_head *secondary_spawns)
 111 {
 112         struct crypto_spawn *spawn, *n;
 113
 114         spawn = list_first_entry_or_null(stack, struct crypto_spawn, list);
 115         if (!spawn)
 116                 return NULL;
 117
 118         n = list_prev_entry(spawn, list);
 119         list_move(&spawn->list, secondary_spawns);
 120
 121         if (list_is_last(&n->list, stack))
 122                 return top;
 123
 124         n = list_next_entry(n, list);
 125         if (!spawn->dead)
 126                 n->dead = false;
 127
 128         return &n->inst->alg.cra_users;
 129 }
 130
 131 static void crypto_remove_instance(struct crypto_instance *inst,
 132                                    struct list_head *list)
 133 {
 134         struct crypto_template *tmpl = inst->tmpl;
 135
 136         if (crypto_is_dead(&inst->alg))
 137                 return;
 138
 139         inst->alg.cra_flags |= CRYPTO_ALG_DEAD;
 140
 141         if (!tmpl || !crypto_tmpl_get(tmpl))
 142                 return;
 143
 144         list_move(&inst->alg.cra_list, list);
 145         hlist_del(&inst->list);
 146         inst->alg.cra_destroy = crypto_destroy_instance;
 147
 148         BUG_ON(!list_empty(&inst->alg.cra_users));
 149 }
 150
 151 /*
 152  * Given an algorithm alg, remove all algorithms that depend on it
 153  * through spawns.  If nalg is not null, then exempt any algorithms
 154  * that is depended on by nalg.  This is useful when nalg itself
 155  * depends on alg.
 156  */
 157 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
 158                           struct crypto_alg *nalg)
 159 {
 160         u32 new_type = (nalg ?: alg)->cra_flags;
 161         struct crypto_spawn *spawn, *n;
 162         LIST_HEAD(secondary_spawns);
 163         struct list_head *spawns;
 164         LIST_HEAD(stack);
 165         LIST_HEAD(top);
 166
 167         spawns = &alg->cra_users;
 168         list_for_each_entry_safe(spawn, n, spawns, list) {
 169                 if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
 170                         continue;
 171
 172                 list_move(&spawn->list, &top);
 173         }
 174
 175         /*
 176          * Perform a depth-first walk starting from alg through
 177          * the cra_users tree.  The list stack records the path
 178          * from alg to the current spawn.
 179          */
 180         spawns = &top;
 181         do {
 182                 while (!list_empty(spawns)) {
 183                         struct crypto_instance *inst;
 184
 185                         spawn = list_first_entry(spawns, struct crypto_spawn,
 186                                                  list);
 187                         inst = spawn->inst;
 188
 189                         list_move(&spawn->list, &stack);
 190                         spawn->dead = !spawn->registered || &inst->alg != nalg;
 191
 192                         if (!spawn->registered)
 193                                 break;
 194
 195                         BUG_ON(&inst->alg == alg);
 196
 197                         if (&inst->alg == nalg)
 198                                 break;
 199
 200                         spawns = &inst->alg.cra_users;
 201
 202                         /*
 203                          * Even if spawn->registered is true, the
 204                          * instance itself may still be unregistered.
 205                          * This is because it may have failed during
 206                          * registration.  Therefore we still need to
 207                          * make the following test.
 208                          *
 209                          * We may encounter an unregistered instance here, since
 210                          * an instance's spawns are set up prior to the instance
 211                          * being registered.  An unregistered instance will have
 212                          * NULL ->cra_users.next, since ->cra_users isn't
 213                          * properly initialized until registration.  But an
 214                          * unregistered instance cannot have any users, so treat
 215                          * it the same as ->cra_users being empty.
 216                          */
 217                         if (spawns->next == NULL)
 218                                 break;
 219                 }
 220         } while ((spawns = crypto_more_spawns(alg, &stack, &top,
 221                                               &secondary_spawns)));
 222
 223         /*
 224          * Remove all instances that are marked as dead.  Also
 225          * complete the resurrection of the others by moving them
 226          * back to the cra_users list.
 227          */
 228         list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
 229                 if (!spawn->dead)
 230                         list_move(&spawn->list, &spawn->alg->cra_users);
 231                 else if (spawn->registered)
 232                         crypto_remove_instance(spawn->inst, list);
 233         }
 234 }
 235 EXPORT_SYMBOL_GPL(crypto_remove_spawns);
 236
 237 static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)
 238 {
 239         struct crypto_larval *larval;
 240
 241         if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER))
 242                 return NULL;
 243
 244         larval = crypto_larval_alloc(alg->cra_name,
 245                                      alg->cra_flags | CRYPTO_ALG_TESTED, 0);
 246         if (IS_ERR(larval))
 247                 return larval;
 248
 249         larval->adult = crypto_mod_get(alg);
 250         if (!larval->adult) {
 251                 kfree(larval);
 252                 return ERR_PTR(-ENOENT);
 253         }
 254
 255         refcount_set(&larval->alg.cra_refcnt, 1);
 256         memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
 257                CRYPTO_MAX_ALG_NAME);
 258         larval->alg.cra_priority = alg->cra_priority;
 259
 260         return larval;
 261 }
 262
 263 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)
 264 {
 265         struct crypto_alg *q;
 266         struct crypto_larval *larval;
 267         int ret = -EAGAIN;
 268
 269         if (crypto_is_dead(alg))
 270                 goto err;
 271
 272         INIT_LIST_HEAD(&alg->cra_users);
 273
 274         /* No cheating! */
 275         alg->cra_flags &= ~CRYPTO_ALG_TESTED;
 276
 277         ret = -EEXIST;
 278
 279         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 280                 if (q == alg)
 281                         goto err;
 282
 283                 if (crypto_is_moribund(q))
 284                         continue;
 285
 286                 if (crypto_is_larval(q)) {
 287                         if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
 288                                 goto err;
 289                         continue;
 290                 }
 291
 292                 if (!strcmp(q->cra_driver_name, alg->cra_name) ||
 293                     !strcmp(q->cra_name, alg->cra_driver_name))
 294                         goto err;
 295         }
 296
 297         larval = crypto_alloc_test_larval(alg);
 298         if (IS_ERR(larval))
 299                 goto out;
 300
 301         list_add(&alg->cra_list, &crypto_alg_list);
 302
 303         if (larval)
 304                 list_add(&larval->alg.cra_list, &crypto_alg_list);
 305         else
 306                 alg->cra_flags |= CRYPTO_ALG_TESTED;
 307
 308         crypto_stats_init(alg);
 309
 310 out:
 311         return larval;
 312
 313 err:
 314         larval = ERR_PTR(ret);
 315         goto out;
 316 }
 317
 318 void crypto_alg_tested(const char *name, int err)
 319 {
 320         struct crypto_larval *test;
 321         struct crypto_alg *alg;
 322         struct crypto_alg *q;
 323         LIST_HEAD(list);
 324         bool best;
 325
 326         down_write(&crypto_alg_sem);
 327         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 328                 if (crypto_is_moribund(q) || !crypto_is_larval(q))
 329                         continue;
 330
 331                 test = (struct crypto_larval *)q;
 332
 333                 if (!strcmp(q->cra_driver_name, name))
 334                         goto found;
 335         }
 336
 337         pr_err("alg: Unexpected test result for %s: %d\n", name, err);
 338         goto unlock;
 339
 340 found:
 341         q->cra_flags |= CRYPTO_ALG_DEAD;
 342         alg = test->adult;
 343
 344         if (list_empty(&alg->cra_list))
 345                 goto complete;
 346
 347         if (err == -ECANCELED)
 348                 alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;
 349         else if (err)
 350                 goto complete;
 351         else
 352                 alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;
 353
 354         alg->cra_flags |= CRYPTO_ALG_TESTED;
 355
 356         /* Only satisfy larval waiters if we are the best. */
 357         best = true;
 358         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 359                 if (crypto_is_moribund(q) || !crypto_is_larval(q))
 360                         continue;
 361
 362                 if (strcmp(alg->cra_name, q->cra_name))
 363                         continue;
 364
 365                 if (q->cra_priority > alg->cra_priority) {
 366                         best = false;
 367                         break;
 368                 }
 369         }
 370
 371         list_for_each_entry(q, &crypto_alg_list, cra_list) {
 372                 if (q == alg)
 373                         continue;
 374
 375                 if (crypto_is_moribund(q))
 376                         continue;
 377
 378                 if (crypto_is_larval(q)) {
 379                         struct crypto_larval *larval = (void *)q;
 380
 381                         /*
 382                          * Check to see if either our generic name or
 383                          * specific name can satisfy the name requested
 384                          * by the larval entry q.
 385                          */
 386                         if (strcmp(alg->cra_name, q->cra_name) &&
 387                             strcmp(alg->cra_driver_name, q->cra_name))
 388                                 continue;
 389
 390                         if (larval->adult)
 391                                 continue;
 392                         if ((q->cra_flags ^ alg->cra_flags) & larval->mask)
 393                                 continue;
 394
 395                         if (best && crypto_mod_get(alg))
 396                                 larval->adult = alg;
 397                         else
 398                                 larval->adult = ERR_PTR(-EAGAIN);
 399
 400                         continue;
 401                 }
 402
 403                 if (strcmp(alg->cra_name, q->cra_name))
 404                         continue;
 405
 406                 if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
 407                     q->cra_priority > alg->cra_priority)
 408                         continue;
 409
 410                 crypto_remove_spawns(q, &list, alg);
 411         }
 412
 413 complete:
 414         complete_all(&test->completion);
 415
 416 unlock:
 417         up_write(&crypto_alg_sem);
 418
 419         crypto_remove_final(&list);
 420 }
 421 EXPORT_SYMBOL_GPL(crypto_alg_tested);
 422
 423 void crypto_remove_final(struct list_head *list)
 424 {
 425         struct crypto_alg *alg;
 426         struct crypto_alg *n;
 427
 428         list_for_each_entry_safe(alg, n, list, cra_list) {
 429                 list_del_init(&alg->cra_list);
 430                 crypto_alg_put(alg);
 431         }
 432 }
 433 EXPORT_SYMBOL_GPL(crypto_remove_final);
 434
 435 int crypto_register_alg(struct crypto_alg *alg)
 436 {
 437         struct crypto_larval *larval;
 438         bool test_started;
 439         int err;
 440
 441         alg->cra_flags &= ~CRYPTO_ALG_DEAD;
 442         err = crypto_check_alg(alg);
 443         if (err)
 444                 return err;
 445
 446         down_write(&crypto_alg_sem);
 447         larval = __crypto_register_alg(alg);
 448         test_started = static_key_enabled(&crypto_boot_test_finished);
 449         if (!IS_ERR_OR_NULL(larval))
 450                 larval->test_started = test_started;
 451         up_write(&crypto_alg_sem);
 452
 453         if (IS_ERR_OR_NULL(larval))
 454                 return PTR_ERR(larval);
 455
 456         if (test_started)
 457                 crypto_wait_for_test(larval);
 458         return 0;
 459 }
 460 EXPORT_SYMBOL_GPL(crypto_register_alg);
 461
 462 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
 463 {
 464         if (unlikely(list_empty(&alg->cra_list)))
 465                 return -ENOENT;
 466
 467         alg->cra_flags |= CRYPTO_ALG_DEAD;
 468
 469         list_del_init(&alg->cra_list);
 470         crypto_remove_spawns(alg, list, NULL);
 471
 472         return 0;
 473 }
 474
 475 void crypto_unregister_alg(struct crypto_alg *alg)
 476 {
 477         int ret;
 478         LIST_HEAD(list);
 479
 480         down_write(&crypto_alg_sem);
 481         ret = crypto_remove_alg(alg, &list);
 482         up_write(&crypto_alg_sem);
 483
 484         if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
 485                 return;
 486
 487         if (WARN_ON(refcount_read(&alg->cra_refcnt) != 1))
 488                 return;
 489
 490         if (alg->cra_destroy)
 491                 alg->cra_destroy(alg);
 492
 493         crypto_remove_final(&list);
 494 }
 495 EXPORT_SYMBOL_GPL(crypto_unregister_alg);
 496
 497 int crypto_register_algs(struct crypto_alg *algs, int count)
 498 {
 499         int i, ret;
 500
 501         for (i = 0; i < count; i++) {
 502                 ret = crypto_register_alg(&algs[i]);
 503                 if (ret)
 504                         goto err;
 505         }
 506
 507         return 0;
 508
 509 err:
 510         for (--i; i >= 0; --i)
 511                 crypto_unregister_alg(&algs[i]);
 512
 513         return ret;
 514 }
 515 EXPORT_SYMBOL_GPL(crypto_register_algs);
 516
 517 void crypto_unregister_algs(struct crypto_alg *algs, int count)
 518 {
 519         int i;
 520
 521         for (i = 0; i < count; i++)
 522                 crypto_unregister_alg(&algs[i]);
 523 }
 524 EXPORT_SYMBOL_GPL(crypto_unregister_algs);
 525
 526 int crypto_register_template(struct crypto_template *tmpl)
 527 {
 528         struct crypto_template *q;
 529         int err = -EEXIST;
 530
 531         down_write(&crypto_alg_sem);
 532
 533         crypto_check_module_sig(tmpl->module);
 534
 535         list_for_each_entry(q, &crypto_template_list, list) {
 536                 if (q == tmpl)
 537                         goto out;
 538         }
 539
 540         list_add(&tmpl->list, &crypto_template_list);
 541         err = 0;
 542 out:
 543         up_write(&crypto_alg_sem);
 544         return err;
 545 }
 546 EXPORT_SYMBOL_GPL(crypto_register_template);
 547
 548 int crypto_register_templates(struct crypto_template *tmpls, int count)
 549 {
 550         int i, err;
 551
 552         for (i = 0; i < count; i++) {
 553                 err = crypto_register_template(&tmpls[i]);
 554                 if (err)
 555                         goto out;
 556         }
 557         return 0;
 558
 559 out:
 560         for (--i; i >= 0; --i)
 561                 crypto_unregister_template(&tmpls[i]);
 562         return err;
 563 }
 564 EXPORT_SYMBOL_GPL(crypto_register_templates);
 565
 566 void crypto_unregister_template(struct crypto_template *tmpl)
 567 {
 568         struct crypto_instance *inst;
 569         struct hlist_node *n;
 570         struct hlist_head *list;
 571         LIST_HEAD(users);
 572
 573         down_write(&crypto_alg_sem);
 574
 575         BUG_ON(list_empty(&tmpl->list));
 576         list_del_init(&tmpl->list);
 577
 578         list = &tmpl->instances;
 579         hlist_for_each_entry(inst, list, list) {
 580                 int err = crypto_remove_alg(&inst->alg, &users);
 581
 582                 BUG_ON(err);
 583         }
 584
 585         up_write(&crypto_alg_sem);
 586
 587         hlist_for_each_entry_safe(inst, n, list, list) {
 588                 BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
 589                 crypto_free_instance(inst);
 590         }
 591         crypto_remove_final(&users);
 592 }
 593 EXPORT_SYMBOL_GPL(crypto_unregister_template);
 594
 595 void crypto_unregister_templates(struct crypto_template *tmpls, int count)
 596 {
 597         int i;
 598
 599         for (i = count - 1; i >= 0; --i)
 600                 crypto_unregister_template(&tmpls[i]);
 601 }
 602 EXPORT_SYMBOL_GPL(crypto_unregister_templates);
 603
 604 static struct crypto_template *__crypto_lookup_template(const char *name)
 605 {
 606         struct crypto_template *q, *tmpl = NULL;
 607
 608         down_read(&crypto_alg_sem);
 609         list_for_each_entry(q, &crypto_template_list, list) {
 610                 if (strcmp(q->name, name))
 611                         continue;
 612                 if (unlikely(!crypto_tmpl_get(q)))
 613                         continue;
 614
 615                 tmpl = q;
 616                 break;
 617         }
 618         up_read(&crypto_alg_sem);
 619
 620         return tmpl;
 621 }
 622
 623 struct crypto_template *crypto_lookup_template(const char *name)
 624 {
 625         return try_then_request_module(__crypto_lookup_template(name),
 626                                        "crypto-%s", name);
 627 }
 628 EXPORT_SYMBOL_GPL(crypto_lookup_template);
 629
 630 int crypto_register_instance(struct crypto_template *tmpl,
 631                              struct crypto_instance *inst)
 632 {
 633         struct crypto_larval *larval;
 634         struct crypto_spawn *spawn;
 635         u32 fips_internal = 0;
 636         int err;
 637
 638         err = crypto_check_alg(&inst->alg);
 639         if (err)
 640                 return err;
 641
 642         inst->alg.cra_module = tmpl->module;
 643         inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
 644
 645         down_write(&crypto_alg_sem);
 646
 647         larval = ERR_PTR(-EAGAIN);
 648         for (spawn = inst->spawns; spawn;) {
 649                 struct crypto_spawn *next;
 650
 651                 if (spawn->dead)
 652                         goto unlock;
 653
 654                 next = spawn->next;
 655                 spawn->inst = inst;
 656                 spawn->registered = true;
 657
 658                 fips_internal |= spawn->alg->cra_flags;
 659
 660                 crypto_mod_put(spawn->alg);
 661
 662                 spawn = next;
 663         }
 664
 665         inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);
 666
 667         larval = __crypto_register_alg(&inst->alg);
 668         if (IS_ERR(larval))
 669                 goto unlock;
 670         else if (larval)
 671                 larval->test_started = true;
 672
 673         hlist_add_head(&inst->list, &tmpl->instances);
 674         inst->tmpl = tmpl;
 675
 676 unlock:
 677         up_write(&crypto_alg_sem);
 678
 679         err = PTR_ERR(larval);
 680         if (IS_ERR_OR_NULL(larval))
 681                 goto err;
 682
 683         crypto_wait_for_test(larval);
 684         err = 0;
 685
 686 err:
 687         return err;
 688 }
 689 EXPORT_SYMBOL_GPL(crypto_register_instance);
 690
 691 void crypto_unregister_instance(struct crypto_instance *inst)
 692 {
 693         LIST_HEAD(list);
 694
 695         down_write(&crypto_alg_sem);
 696
 697         crypto_remove_spawns(&inst->alg, &list, NULL);
 698         crypto_remove_instance(inst, &list);
 699
 700         up_write(&crypto_alg_sem);
 701
 702         crypto_remove_final(&list);
 703 }
 704 EXPORT_SYMBOL_GPL(crypto_unregister_instance);
 705
 706 int crypto_grab_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst,
 707                       const char *name, u32 type, u32 mask)
 708 {
 709         struct crypto_alg *alg;
 710         int err = -EAGAIN;
 711
 712         if (WARN_ON_ONCE(inst == NULL))
 713                 return -EINVAL;
 714
 715         /* Allow the result of crypto_attr_alg_name() to be passed directly */
 716         if (IS_ERR(name))
 717                 return PTR_ERR(name);
 718
 719         alg = crypto_find_alg(name, spawn->frontend,
 720                               type | CRYPTO_ALG_FIPS_INTERNAL, mask);
 721         if (IS_ERR(alg))
 722                 return PTR_ERR(alg);
 723
 724         down_write(&crypto_alg_sem);
 725         if (!crypto_is_moribund(alg)) {
 726                 list_add(&spawn->list, &alg->cra_users);
 727                 spawn->alg = alg;
 728                 spawn->mask = mask;
 729                 spawn->next = inst->spawns;
 730                 inst->spawns = spawn;
 731                 inst->alg.cra_flags |=
 732                         (alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
 733                 err = 0;
 734         }
 735         up_write(&crypto_alg_sem);
 736         if (err)
 737                 crypto_mod_put(alg);
 738         return err;
 739 }
 740 EXPORT_SYMBOL_GPL(crypto_grab_spawn);
 741
 742 void crypto_drop_spawn(struct crypto_spawn *spawn)
 743 {
 744         if (!spawn->alg) /* not yet initialized? */
 745                 return;
 746
 747         down_write(&crypto_alg_sem);
 748         if (!spawn->dead)
 749                 list_del(&spawn->list);
 750         up_write(&crypto_alg_sem);
 751
 752         if (!spawn->registered)
 753                 crypto_mod_put(spawn->alg);
 754 }
 755 EXPORT_SYMBOL_GPL(crypto_drop_spawn);
 756
 757 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn)
 758 {
 759         struct crypto_alg *alg = ERR_PTR(-EAGAIN);
 760         struct crypto_alg *target;
 761         bool shoot = false;
 762
 763         down_read(&crypto_alg_sem);
 764         if (!spawn->dead) {
 765                 alg = spawn->alg;
 766                 if (!crypto_mod_get(alg)) {
 767                         target = crypto_alg_get(alg);
 768                         shoot = true;
 769                         alg = ERR_PTR(-EAGAIN);
 770                 }
 771         }
 772         up_read(&crypto_alg_sem);
 773
 774         if (shoot) {
 775                 crypto_shoot_alg(target);
 776                 crypto_alg_put(target);
 777         }
 778
 779         return alg;
 780 }
 781
 782 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type,
 783                                     u32 mask)
 784 {
 785         struct crypto_alg *alg;
 786         struct crypto_tfm *tfm;
 787
 788         alg = crypto_spawn_alg(spawn);
 789         if (IS_ERR(alg))
 790                 return ERR_CAST(alg);
 791
 792         tfm = ERR_PTR(-EINVAL);
 793         if (unlikely((alg->cra_flags ^ type) & mask))
 794                 goto out_put_alg;
 795
 796         tfm = __crypto_alloc_tfm(alg, type, mask);
 797         if (IS_ERR(tfm))
 798                 goto out_put_alg;
 799
 800         return tfm;
 801
 802 out_put_alg:
 803         crypto_mod_put(alg);
 804         return tfm;
 805 }
 806 EXPORT_SYMBOL_GPL(crypto_spawn_tfm);
 807
 808 void *crypto_spawn_tfm2(struct crypto_spawn *spawn)
 809 {
 810         struct crypto_alg *alg;
 811         struct crypto_tfm *tfm;
 812
 813         alg = crypto_spawn_alg(spawn);
 814         if (IS_ERR(alg))
 815                 return ERR_CAST(alg);
 816
 817         tfm = crypto_create_tfm(alg, spawn->frontend);
 818         if (IS_ERR(tfm))
 819                 goto out_put_alg;
 820
 821         return tfm;
 822
 823 out_put_alg:
 824         crypto_mod_put(alg);
 825         return tfm;
 826 }
 827 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);
 828
 829 int crypto_register_notifier(struct notifier_block *nb)
 830 {
 831         return blocking_notifier_chain_register(&crypto_chain, nb);
 832 }
 833 EXPORT_SYMBOL_GPL(crypto_register_notifier);
 834
 835 int crypto_unregister_notifier(struct notifier_block *nb)
 836 {
 837         return blocking_notifier_chain_unregister(&crypto_chain, nb);
 838 }
 839 EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
 840
 841 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
 842 {
 843         struct rtattr *rta = tb[0];
 844         struct crypto_attr_type *algt;
 845
 846         if (!rta)
 847                 return ERR_PTR(-ENOENT);
 848         if (RTA_PAYLOAD(rta) < sizeof(*algt))
 849                 return ERR_PTR(-EINVAL);
 850         if (rta->rta_type != CRYPTOA_TYPE)
 851                 return ERR_PTR(-EINVAL);
 852
 853         algt = RTA_DATA(rta);
 854
 855         return algt;
 856 }
 857 EXPORT_SYMBOL_GPL(crypto_get_attr_type);
 858
 859 /**
 860  * crypto_check_attr_type() - check algorithm type and compute inherited mask
 861  * @tb: the template parameters
 862  * @type: the algorithm type the template would be instantiated as
 863  * @mask_ret: (output) the mask that should be passed to crypto_grab_*()
 864  *            to restrict the flags of any inner algorithms
 865  *
 866  * Validate that the algorithm type the user requested is compatible with the
 867  * one the template would actually be instantiated as.  E.g., if the user is
 868  * doing crypto_alloc_shash("cbc(aes)", ...), this would return an error because
 869  * the "cbc" template creates an "skcipher" algorithm, not an "shash" algorithm.
 870  *
 871  * Also compute the mask to use to restrict the flags of any inner algorithms.
 872  *
 873  * Return: 0 on success; -errno on failure
 874  */
 875 int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret)
 876 {
 877         struct crypto_attr_type *algt;
 878
 879         algt = crypto_get_attr_type(tb);
 880         if (IS_ERR(algt))
 881                 return PTR_ERR(algt);
 882
 883         if ((algt->type ^ type) & algt->mask)
 884                 return -EINVAL;
 885
 886         *mask_ret = crypto_algt_inherited_mask(algt);
 887         return 0;
 888 }
 889 EXPORT_SYMBOL_GPL(crypto_check_attr_type);
 890
 891 const char *crypto_attr_alg_name(struct rtattr *rta)
 892 {
 893         struct crypto_attr_alg *alga;
 894
 895         if (!rta)
 896                 return ERR_PTR(-ENOENT);
 897         if (RTA_PAYLOAD(rta) < sizeof(*alga))
 898                 return ERR_PTR(-EINVAL);
 899         if (rta->rta_type != CRYPTOA_ALG)
 900                 return ERR_PTR(-EINVAL);
 901
 902         alga = RTA_DATA(rta);
 903         alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
 904
 905         return alga->name;
 906 }
 907 EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
 908
 909 int crypto_inst_setname(struct crypto_instance *inst, const char *name,
 910                         struct crypto_alg *alg)
 911 {
 912         if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
 913                      alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
 914                 return -ENAMETOOLONG;
 915
 916         if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
 917                      name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 918                 return -ENAMETOOLONG;
 919
 920         return 0;
 921 }
 922 EXPORT_SYMBOL_GPL(crypto_inst_setname);
 923
 924 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
 925 {
 926         INIT_LIST_HEAD(&queue->list);
 927         queue->backlog = &queue->list;
 928         queue->qlen = 0;
 929         queue->max_qlen = max_qlen;
 930 }
 931 EXPORT_SYMBOL_GPL(crypto_init_queue);
 932
 933 int crypto_enqueue_request(struct crypto_queue *queue,
 934                            struct crypto_async_request *request)
 935 {
 936         int err = -EINPROGRESS;
 937
 938         if (unlikely(queue->qlen >= queue->max_qlen)) {
 939                 if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 940                         err = -ENOSPC;
 941                         goto out;
 942                 }
 943                 err = -EBUSY;
 944                 if (queue->backlog == &queue->list)
 945                         queue->backlog = &request->list;
 946         }
 947
 948         queue->qlen++;
 949         list_add_tail(&request->list, &queue->list);
 950
 951 out:
 952         return err;
 953 }
 954 EXPORT_SYMBOL_GPL(crypto_enqueue_request);
 955
 956 void crypto_enqueue_request_head(struct crypto_queue *queue,
 957                                  struct crypto_async_request *request)
 958 {
 959         if (unlikely(queue->qlen >= queue->max_qlen))
 960                 queue->backlog = queue->backlog->prev;
 961
 962         queue->qlen++;
 963         list_add(&request->list, &queue->list);
 964 }
 965 EXPORT_SYMBOL_GPL(crypto_enqueue_request_head);
 966
 967 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
 968 {
 969         struct list_head *request;
 970
 971         if (unlikely(!queue->qlen))
 972                 return NULL;
 973
 974         queue->qlen--;
 975
 976         if (queue->backlog != &queue->list)
 977                 queue->backlog = queue->backlog->next;
 978
 979         request = queue->list.next;
 980         list_del(request);
 981
 982         return list_entry(request, struct crypto_async_request, list);
 983 }
 984 EXPORT_SYMBOL_GPL(crypto_dequeue_request);
 985
 986 static inline void crypto_inc_byte(u8 *a, unsigned int size)
 987 {
 988         u8 *b = (a + size);
 989         u8 c;
 990
 991         for (; size; size--) {
 992                 c = *--b + 1;
 993                 *b = c;
 994                 if (c)
 995                         break;
 996         }
 997 }
 998
 999 void crypto_inc(u8 *a, unsigned int size)
1000 {
1001         __be32 *b = (__be32 *)(a + size);
1002         u32 c;
1003
1004         if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
1005             IS_ALIGNED((unsigned long)b, __alignof__(*b)))
1006                 for (; size >= 4; size -= 4) {
1007                         c = be32_to_cpu(*--b) + 1;
1008                         *b = cpu_to_be32(c);
1009                         if (likely(c))
1010                                 return;
1011                 }
1012
1013         crypto_inc_byte(a, size);
1014 }
1015 EXPORT_SYMBOL_GPL(crypto_inc);
1016
1017 unsigned int crypto_alg_extsize(struct crypto_alg *alg)
1018 {
1019         return alg->cra_ctxsize +
1020                (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
1021 }
1022 EXPORT_SYMBOL_GPL(crypto_alg_extsize);
1023
1024 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
1025                         u32 type, u32 mask)
1026 {
1027         int ret = 0;
1028         struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);
1029
1030         if (!IS_ERR(alg)) {
1031                 crypto_mod_put(alg);
1032                 ret = 1;
1033         }
1034
1035         return ret;
1036 }
1037 EXPORT_SYMBOL_GPL(crypto_type_has_alg);
1038
1039 #ifdef CONFIG_CRYPTO_STATS
1040 void crypto_stats_init(struct crypto_alg *alg)
1041 {
1042         memset(&alg->stats, 0, sizeof(alg->stats));
1043 }
1044 EXPORT_SYMBOL_GPL(crypto_stats_init);
1045
1046 void crypto_stats_get(struct crypto_alg *alg)
1047 {
1048         crypto_alg_get(alg);
1049 }
1050 EXPORT_SYMBOL_GPL(crypto_stats_get);
1051
1052 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg,
1053                                int ret)
1054 {
1055         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1056                 atomic64_inc(&alg->stats.aead.err_cnt);
1057         } else {
1058                 atomic64_inc(&alg->stats.aead.encrypt_cnt);
1059                 atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen);
1060         }
1061         crypto_alg_put(alg);
1062 }
1063 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt);
1064
1065 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg,
1066                                int ret)
1067 {
1068         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1069                 atomic64_inc(&alg->stats.aead.err_cnt);
1070         } else {
1071                 atomic64_inc(&alg->stats.aead.decrypt_cnt);
1072                 atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen);
1073         }
1074         crypto_alg_put(alg);
1075 }
1076 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt);
1077
1078 void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret,
1079                                    struct crypto_alg *alg)
1080 {
1081         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1082                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1083         } else {
1084                 atomic64_inc(&alg->stats.akcipher.encrypt_cnt);
1085                 atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen);
1086         }
1087         crypto_alg_put(alg);
1088 }
1089 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt);
1090
1091 void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret,
1092                                    struct crypto_alg *alg)
1093 {
1094         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1095                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1096         } else {
1097                 atomic64_inc(&alg->stats.akcipher.decrypt_cnt);
1098                 atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen);
1099         }
1100         crypto_alg_put(alg);
1101 }
1102 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt);
1103
1104 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg)
1105 {
1106         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1107                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1108         else
1109                 atomic64_inc(&alg->stats.akcipher.sign_cnt);
1110         crypto_alg_put(alg);
1111 }
1112 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign);
1113
1114 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg)
1115 {
1116         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1117                 atomic64_inc(&alg->stats.akcipher.err_cnt);
1118         else
1119                 atomic64_inc(&alg->stats.akcipher.verify_cnt);
1120         crypto_alg_put(alg);
1121 }
1122 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify);
1123
1124 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg)
1125 {
1126         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1127                 atomic64_inc(&alg->stats.compress.err_cnt);
1128         } else {
1129                 atomic64_inc(&alg->stats.compress.compress_cnt);
1130                 atomic64_add(slen, &alg->stats.compress.compress_tlen);
1131         }
1132         crypto_alg_put(alg);
1133 }
1134 EXPORT_SYMBOL_GPL(crypto_stats_compress);
1135
1136 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg)
1137 {
1138         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1139                 atomic64_inc(&alg->stats.compress.err_cnt);
1140         } else {
1141                 atomic64_inc(&alg->stats.compress.decompress_cnt);
1142                 atomic64_add(slen, &alg->stats.compress.decompress_tlen);
1143         }
1144         crypto_alg_put(alg);
1145 }
1146 EXPORT_SYMBOL_GPL(crypto_stats_decompress);
1147
1148 void crypto_stats_ahash_update(unsigned int nbytes, int ret,
1149                                struct crypto_alg *alg)
1150 {
1151         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1152                 atomic64_inc(&alg->stats.hash.err_cnt);
1153         else
1154                 atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
1155         crypto_alg_put(alg);
1156 }
1157 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update);
1158
1159 void crypto_stats_ahash_final(unsigned int nbytes, int ret,
1160                               struct crypto_alg *alg)
1161 {
1162         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1163                 atomic64_inc(&alg->stats.hash.err_cnt);
1164         } else {
1165                 atomic64_inc(&alg->stats.hash.hash_cnt);
1166                 atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
1167         }
1168         crypto_alg_put(alg);
1169 }
1170 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final);
1171
1172 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret)
1173 {
1174         if (ret)
1175                 atomic64_inc(&alg->stats.kpp.err_cnt);
1176         else
1177                 atomic64_inc(&alg->stats.kpp.setsecret_cnt);
1178         crypto_alg_put(alg);
1179 }
1180 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret);
1181
1182 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret)
1183 {
1184         if (ret)
1185                 atomic64_inc(&alg->stats.kpp.err_cnt);
1186         else
1187                 atomic64_inc(&alg->stats.kpp.generate_public_key_cnt);
1188         crypto_alg_put(alg);
1189 }
1190 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key);
1191
1192 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret)
1193 {
1194         if (ret)
1195                 atomic64_inc(&alg->stats.kpp.err_cnt);
1196         else
1197                 atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt);
1198         crypto_alg_put(alg);
1199 }
1200 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret);
1201
1202 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret)
1203 {
1204         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1205                 atomic64_inc(&alg->stats.rng.err_cnt);
1206         else
1207                 atomic64_inc(&alg->stats.rng.seed_cnt);
1208         crypto_alg_put(alg);
1209 }
1210 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed);
1211
1212 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen,
1213                                int ret)
1214 {
1215         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1216                 atomic64_inc(&alg->stats.rng.err_cnt);
1217         } else {
1218                 atomic64_inc(&alg->stats.rng.generate_cnt);
1219                 atomic64_add(dlen, &alg->stats.rng.generate_tlen);
1220         }
1221         crypto_alg_put(alg);
1222 }
1223 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate);
1224
1225 void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret,
1226                                    struct crypto_alg *alg)
1227 {
1228         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1229                 atomic64_inc(&alg->stats.cipher.err_cnt);
1230         } else {
1231                 atomic64_inc(&alg->stats.cipher.encrypt_cnt);
1232                 atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen);
1233         }
1234         crypto_alg_put(alg);
1235 }
1236 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt);
1237
1238 void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret,
1239                                    struct crypto_alg *alg)
1240 {
1241         if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1242                 atomic64_inc(&alg->stats.cipher.err_cnt);
1243         } else {
1244                 atomic64_inc(&alg->stats.cipher.decrypt_cnt);
1245                 atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen);
1246         }
1247         crypto_alg_put(alg);
1248 }
1249 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt);
1250 #endif
1251
1252 static void __init crypto_start_tests(void)
1253 {
1254         for (;;) {
1255                 struct crypto_larval *larval = NULL;
1256                 struct crypto_alg *q;
1257
1258                 down_write(&crypto_alg_sem);
1259
1260                 list_for_each_entry(q, &crypto_alg_list, cra_list) {
1261                         struct crypto_larval *l;
1262
1263                         if (!crypto_is_larval(q))
1264                                 continue;
1265
1266                         l = (void *)q;
1267
1268                         if (!crypto_is_test_larval(l))
1269                                 continue;
1270
1271                         if (l->test_started)
1272                                 continue;
1273
1274                         l->test_started = true;
1275                         larval = l;
1276                         break;
1277                 }
1278
1279                 up_write(&crypto_alg_sem);
1280
1281                 if (!larval)
1282                         break;
1283
1284                 crypto_wait_for_test(larval);
1285         }
1286
1287         static_branch_enable(&crypto_boot_test_finished);
1288 }
1289
1290 static int __init crypto_algapi_init(void)
1291 {
1292         crypto_init_proc();
1293         crypto_start_tests();
1294         return 0;
1295 }
1296
1297 static void __exit crypto_algapi_exit(void)
1298 {
1299         crypto_exit_proc();
1300 }
1301
1302 /*
1303  * We run this at late_initcall so that all the built-in algorithms
1304  * have had a chance to register themselves first.
1305  */
1306 late_initcall(crypto_algapi_init);
1307 module_exit(crypto_algapi_exit);
1308
1309 MODULE_LICENSE("GPL");
1310 MODULE_DESCRIPTION("Cryptographic algorithms API");
1311 MODULE_SOFTDEP("pre: cryptomgr");