crypto/crypto_engine.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Handle async block request by crypto hardware engine.
   4  *
   5  * Copyright (C) 2016 Linaro, Inc.
   6  *
   7  * Author: Baolin Wang <baolin.wang@linaro.org>
   8  */
   9
  10 #include <linux/err.h>
  11 #include <linux/delay.h>
  12 #include <linux/device.h>
  13 #include <crypto/engine.h>
  14 #include <uapi/linux/sched/types.h>
  15 #include "internal.h"
  16
  17 #define CRYPTO_ENGINE_MAX_QLEN 10
  18
  19 /**
  20  * crypto_finalize_request - finalize one request if the request is done
  21  * @engine: the hardware engine
  22  * @req: the request need to be finalized
  23  * @err: error number
  24  */
  25 static void crypto_finalize_request(struct crypto_engine *engine,
  26                                     struct crypto_async_request *req, int err)
  27 {
  28         unsigned long flags;
  29         bool finalize_req = false;
  30         int ret;
  31         struct crypto_engine_ctx *enginectx;
  32
  33         /*
  34          * If hardware cannot enqueue more requests
  35          * and retry mechanism is not supported
  36          * make sure we are completing the current request
  37          */
  38         if (!engine->retry_support) {
  39                 spin_lock_irqsave(&engine->queue_lock, flags);
  40                 if (engine->cur_req == req) {
  41                         finalize_req = true;
  42                         engine->cur_req = NULL;
  43                 }
  44                 spin_unlock_irqrestore(&engine->queue_lock, flags);
  45         }
  46
  47         if (finalize_req || engine->retry_support) {
  48                 enginectx = crypto_tfm_ctx(req->tfm);
  49                 if (enginectx->op.prepare_request &&
  50                     enginectx->op.unprepare_request) {
  51                         ret = enginectx->op.unprepare_request(engine, req);
  52                         if (ret)
  53                                 dev_err(engine->dev, "failed to unprepare request\n");
  54                 }
  55         }
  56         req->complete(req, err);
  57
  58         kthread_queue_work(engine->kworker, &engine->pump_requests);
  59 }
  60
  61 /**
  62  * crypto_pump_requests - dequeue one request from engine queue to process
  63  * @engine: the hardware engine
  64  * @in_kthread: true if we are in the context of the request pump thread
  65  *
  66  * This function checks if there is any request in the engine queue that
  67  * needs processing and if so call out to the driver to initialize hardware
  68  * and handle each request.
  69  */
  70 static void crypto_pump_requests(struct crypto_engine *engine,
  71                                  bool in_kthread)
  72 {
  73         struct crypto_async_request *async_req, *backlog;
  74         unsigned long flags;
  75         bool was_busy = false;
  76         int ret;
  77         struct crypto_engine_ctx *enginectx;
  78
  79         spin_lock_irqsave(&engine->queue_lock, flags);
  80
  81         /* Make sure we are not already running a request */
  82         if (!engine->retry_support && engine->cur_req)
  83                 goto out;
  84
  85         /* If another context is idling then defer */
  86         if (engine->idling) {
  87                 kthread_queue_work(engine->kworker, &engine->pump_requests);
  88                 goto out;
  89         }
  90
  91         /* Check if the engine queue is idle */
  92         if (!crypto_queue_len(&engine->queue) || !engine->running) {
  93                 if (!engine->busy)
  94                         goto out;
  95
  96                 /* Only do teardown in the thread */
  97                 if (!in_kthread) {
  98                         kthread_queue_work(engine->kworker,
  99                                            &engine->pump_requests);
 100                         goto out;
 101                 }
 102
 103                 engine->busy = false;
 104                 engine->idling = true;
 105                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 106
 107                 if (engine->unprepare_crypt_hardware &&
 108                     engine->unprepare_crypt_hardware(engine))
 109                         dev_err(engine->dev, "failed to unprepare crypt hardware\n");
 110
 111                 spin_lock_irqsave(&engine->queue_lock, flags);
 112                 engine->idling = false;
 113                 goto out;
 114         }
 115
 116 start_request:
 117         /* Get the fist request from the engine queue to handle */
 118         backlog = crypto_get_backlog(&engine->queue);
 119         async_req = crypto_dequeue_request(&engine->queue);
 120         if (!async_req)
 121                 goto out;
 122
 123         /*
 124          * If hardware doesn't support the retry mechanism,
 125          * keep track of the request we are processing now.
 126          * We'll need it on completion (crypto_finalize_request).
 127          */
 128         if (!engine->retry_support)
 129                 engine->cur_req = async_req;
 130
 131         if (backlog)
 132                 backlog->complete(backlog, -EINPROGRESS);
 133
 134         if (engine->busy)
 135                 was_busy = true;
 136         else
 137                 engine->busy = true;
 138
 139         spin_unlock_irqrestore(&engine->queue_lock, flags);
 140
 141         /* Until here we get the request need to be encrypted successfully */
 142         if (!was_busy && engine->prepare_crypt_hardware) {
 143                 ret = engine->prepare_crypt_hardware(engine);
 144                 if (ret) {
 145                         dev_err(engine->dev, "failed to prepare crypt hardware\n");
 146                         goto req_err_2;
 147                 }
 148         }
 149
 150         enginectx = crypto_tfm_ctx(async_req->tfm);
 151
 152         if (enginectx->op.prepare_request) {
 153                 ret = enginectx->op.prepare_request(engine, async_req);
 154                 if (ret) {
 155                         dev_err(engine->dev, "failed to prepare request: %d\n",
 156                                 ret);
 157                         goto req_err_2;
 158                 }
 159         }
 160         if (!enginectx->op.do_one_request) {
 161                 dev_err(engine->dev, "failed to do request\n");
 162                 ret = -EINVAL;
 163                 goto req_err_1;
 164         }
 165
 166         ret = enginectx->op.do_one_request(engine, async_req);
 167
 168         /* Request unsuccessfully executed by hardware */
 169         if (ret < 0) {
 170                 /*
 171                  * If hardware queue is full (-ENOSPC), requeue request
 172                  * regardless of backlog flag.
 173                  * Otherwise, unprepare and complete the request.
 174                  */
 175                 if (!engine->retry_support ||
 176                     (ret != -ENOSPC)) {
 177                         dev_err(engine->dev,
 178                                 "Failed to do one request from queue: %d\n",
 179                                 ret);
 180                         goto req_err_1;
 181                 }
 182                 /*
 183                  * If retry mechanism is supported,
 184                  * unprepare current request and
 185                  * enqueue it back into crypto-engine queue.
 186                  */
 187                 if (enginectx->op.unprepare_request) {
 188                         ret = enginectx->op.unprepare_request(engine,
 189                                                               async_req);
 190                         if (ret)
 191                                 dev_err(engine->dev,
 192                                         "failed to unprepare request\n");
 193                 }
 194                 spin_lock_irqsave(&engine->queue_lock, flags);
 195                 /*
 196                  * If hardware was unable to execute request, enqueue it
 197                  * back in front of crypto-engine queue, to keep the order
 198                  * of requests.
 199                  */
 200                 crypto_enqueue_request_head(&engine->queue, async_req);
 201
 202                 kthread_queue_work(engine->kworker, &engine->pump_requests);
 203                 goto out;
 204         }
 205
 206         goto retry;
 207
 208 req_err_1:
 209         if (enginectx->op.unprepare_request) {
 210                 ret = enginectx->op.unprepare_request(engine, async_req);
 211                 if (ret)
 212                         dev_err(engine->dev, "failed to unprepare request\n");
 213         }
 214
 215 req_err_2:
 216         async_req->complete(async_req, ret);
 217
 218 retry:
 219         /* If retry mechanism is supported, send new requests to engine */
 220         if (engine->retry_support) {
 221                 spin_lock_irqsave(&engine->queue_lock, flags);
 222                 goto start_request;
 223         }
 224         return;
 225
 226 out:
 227         spin_unlock_irqrestore(&engine->queue_lock, flags);
 228
 229         /*
 230          * Batch requests is possible only if
 231          * hardware can enqueue multiple requests
 232          */
 233         if (engine->do_batch_requests) {
 234                 ret = engine->do_batch_requests(engine);
 235                 if (ret)
 236                         dev_err(engine->dev, "failed to do batch requests: %d\n",
 237                                 ret);
 238         }
 239
 240         return;
 241 }
 242
 243 static void crypto_pump_work(struct kthread_work *work)
 244 {
 245         struct crypto_engine *engine =
 246                 container_of(work, struct crypto_engine, pump_requests);
 247
 248         crypto_pump_requests(engine, true);
 249 }
 250
 251 /**
 252  * crypto_transfer_request - transfer the new request into the engine queue
 253  * @engine: the hardware engine
 254  * @req: the request need to be listed into the engine queue
 255  */
 256 static int crypto_transfer_request(struct crypto_engine *engine,
 257                                    struct crypto_async_request *req,
 258                                    bool need_pump)
 259 {
 260         unsigned long flags;
 261         int ret;
 262
 263         spin_lock_irqsave(&engine->queue_lock, flags);
 264
 265         if (!engine->running) {
 266                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 267                 return -ESHUTDOWN;
 268         }
 269
 270         ret = crypto_enqueue_request(&engine->queue, req);
 271
 272         if (!engine->busy && need_pump)
 273                 kthread_queue_work(engine->kworker, &engine->pump_requests);
 274
 275         spin_unlock_irqrestore(&engine->queue_lock, flags);
 276         return ret;
 277 }
 278
 279 /**
 280  * crypto_transfer_request_to_engine - transfer one request to list
 281  * into the engine queue
 282  * @engine: the hardware engine
 283  * @req: the request need to be listed into the engine queue
 284  */
 285 static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
 286                                              struct crypto_async_request *req)
 287 {
 288         return crypto_transfer_request(engine, req, true);
 289 }
 290
 291 /**
 292  * crypto_transfer_aead_request_to_engine - transfer one aead_request
 293  * to list into the engine queue
 294  * @engine: the hardware engine
 295  * @req: the request need to be listed into the engine queue
 296  */
 297 int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
 298                                            struct aead_request *req)
 299 {
 300         return crypto_transfer_request_to_engine(engine, &req->base);
 301 }
 302 EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
 303
 304 /**
 305  * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
 306  * to list into the engine queue
 307  * @engine: the hardware engine
 308  * @req: the request need to be listed into the engine queue
 309  */
 310 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
 311                                                struct akcipher_request *req)
 312 {
 313         return crypto_transfer_request_to_engine(engine, &req->base);
 314 }
 315 EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
 316
 317 /**
 318  * crypto_transfer_hash_request_to_engine - transfer one ahash_request
 319  * to list into the engine queue
 320  * @engine: the hardware engine
 321  * @req: the request need to be listed into the engine queue
 322  */
 323 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
 324                                            struct ahash_request *req)
 325 {
 326         return crypto_transfer_request_to_engine(engine, &req->base);
 327 }
 328 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
 329
 330 /**
 331  * crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list
 332  * into the engine queue
 333  * @engine: the hardware engine
 334  * @req: the request need to be listed into the engine queue
 335  */
 336 int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,
 337                                           struct kpp_request *req)
 338 {
 339         return crypto_transfer_request_to_engine(engine, &req->base);
 340 }
 341 EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine);
 342
 343 /**
 344  * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
 345  * to list into the engine queue
 346  * @engine: the hardware engine
 347  * @req: the request need to be listed into the engine queue
 348  */
 349 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
 350                                                struct skcipher_request *req)
 351 {
 352         return crypto_transfer_request_to_engine(engine, &req->base);
 353 }
 354 EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
 355
 356 /**
 357  * crypto_finalize_aead_request - finalize one aead_request if
 358  * the request is done
 359  * @engine: the hardware engine
 360  * @req: the request need to be finalized
 361  * @err: error number
 362  */
 363 void crypto_finalize_aead_request(struct crypto_engine *engine,
 364                                   struct aead_request *req, int err)
 365 {
 366         return crypto_finalize_request(engine, &req->base, err);
 367 }
 368 EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
 369
 370 /**
 371  * crypto_finalize_akcipher_request - finalize one akcipher_request if
 372  * the request is done
 373  * @engine: the hardware engine
 374  * @req: the request need to be finalized
 375  * @err: error number
 376  */
 377 void crypto_finalize_akcipher_request(struct crypto_engine *engine,
 378                                       struct akcipher_request *req, int err)
 379 {
 380         return crypto_finalize_request(engine, &req->base, err);
 381 }
 382 EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
 383
 384 /**
 385  * crypto_finalize_hash_request - finalize one ahash_request if
 386  * the request is done
 387  * @engine: the hardware engine
 388  * @req: the request need to be finalized
 389  * @err: error number
 390  */
 391 void crypto_finalize_hash_request(struct crypto_engine *engine,
 392                                   struct ahash_request *req, int err)
 393 {
 394         return crypto_finalize_request(engine, &req->base, err);
 395 }
 396 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
 397
 398 /**
 399  * crypto_finalize_kpp_request - finalize one kpp_request if the request is done
 400  * @engine: the hardware engine
 401  * @req: the request need to be finalized
 402  * @err: error number
 403  */
 404 void crypto_finalize_kpp_request(struct crypto_engine *engine,
 405                                  struct kpp_request *req, int err)
 406 {
 407         return crypto_finalize_request(engine, &req->base, err);
 408 }
 409 EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request);
 410
 411 /**
 412  * crypto_finalize_skcipher_request - finalize one skcipher_request if
 413  * the request is done
 414  * @engine: the hardware engine
 415  * @req: the request need to be finalized
 416  * @err: error number
 417  */
 418 void crypto_finalize_skcipher_request(struct crypto_engine *engine,
 419                                       struct skcipher_request *req, int err)
 420 {
 421         return crypto_finalize_request(engine, &req->base, err);
 422 }
 423 EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
 424
 425 /**
 426  * crypto_engine_start - start the hardware engine
 427  * @engine: the hardware engine need to be started
 428  *
 429  * Return 0 on success, else on fail.
 430  */
 431 int crypto_engine_start(struct crypto_engine *engine)
 432 {
 433         unsigned long flags;
 434
 435         spin_lock_irqsave(&engine->queue_lock, flags);
 436
 437         if (engine->running || engine->busy) {
 438                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 439                 return -EBUSY;
 440         }
 441
 442         engine->running = true;
 443         spin_unlock_irqrestore(&engine->queue_lock, flags);
 444
 445         kthread_queue_work(engine->kworker, &engine->pump_requests);
 446
 447         return 0;
 448 }
 449 EXPORT_SYMBOL_GPL(crypto_engine_start);
 450
 451 /**
 452  * crypto_engine_stop - stop the hardware engine
 453  * @engine: the hardware engine need to be stopped
 454  *
 455  * Return 0 on success, else on fail.
 456  */
 457 int crypto_engine_stop(struct crypto_engine *engine)
 458 {
 459         unsigned long flags;
 460         unsigned int limit = 500;
 461         int ret = 0;
 462
 463         spin_lock_irqsave(&engine->queue_lock, flags);
 464
 465         /*
 466          * If the engine queue is not empty or the engine is on busy state,
 467          * we need to wait for a while to pump the requests of engine queue.
 468          */
 469         while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
 470                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 471                 msleep(20);
 472                 spin_lock_irqsave(&engine->queue_lock, flags);
 473         }
 474
 475         if (crypto_queue_len(&engine->queue) || engine->busy)
 476                 ret = -EBUSY;
 477         else
 478                 engine->running = false;
 479
 480         spin_unlock_irqrestore(&engine->queue_lock, flags);
 481
 482         if (ret)
 483                 dev_warn(engine->dev, "could not stop engine\n");
 484
 485         return ret;
 486 }
 487 EXPORT_SYMBOL_GPL(crypto_engine_stop);
 488
 489 /**
 490  * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
 491  * and initialize it by setting the maximum number of entries in the software
 492  * crypto-engine queue.
 493  * @dev: the device attached with one hardware engine
 494  * @retry_support: whether hardware has support for retry mechanism
 495  * @cbk_do_batch: pointer to a callback function to be invoked when executing
 496  *                a batch of requests.
 497  *                This has the form:
 498  *                callback(struct crypto_engine *engine)
 499  *                where:
 500  *                @engine: the crypto engine structure.
 501  * @rt: whether this queue is set to run as a realtime task
 502  * @qlen: maximum size of the crypto-engine queue
 503  *
 504  * This must be called from context that can sleep.
 505  * Return: the crypto engine structure on success, else NULL.
 506  */
 507 struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
 508                                                        bool retry_support,
 509                                                        int (*cbk_do_batch)(struct crypto_engine *engine),
 510                                                        bool rt, int qlen)
 511 {
 512         struct crypto_engine *engine;
 513
 514         if (!dev)
 515                 return NULL;
 516
 517         engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
 518         if (!engine)
 519                 return NULL;
 520
 521         engine->dev = dev;
 522         engine->rt = rt;
 523         engine->running = false;
 524         engine->busy = false;
 525         engine->idling = false;
 526         engine->retry_support = retry_support;
 527         engine->priv_data = dev;
 528         /*
 529          * Batch requests is possible only if
 530          * hardware has support for retry mechanism.
 531          */
 532         engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
 533
 534         snprintf(engine->name, sizeof(engine->name),
 535                  "%s-engine", dev_name(dev));
 536
 537         crypto_init_queue(&engine->queue, qlen);
 538         spin_lock_init(&engine->queue_lock);
 539
 540         engine->kworker = kthread_create_worker(0, "%s", engine->name);
 541         if (IS_ERR(engine->kworker)) {
 542                 dev_err(dev, "failed to create crypto request pump task\n");
 543                 return NULL;
 544         }
 545         kthread_init_work(&engine->pump_requests, crypto_pump_work);
 546
 547         if (engine->rt) {
 548                 dev_info(dev, "will run requests pump with realtime priority\n");
 549                 sched_set_fifo(engine->kworker->task);
 550         }
 551
 552         return engine;
 553 }
 554 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
 555
 556 /**
 557  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 558  * initialize it.
 559  * @dev: the device attached with one hardware engine
 560  * @rt: whether this queue is set to run as a realtime task
 561  *
 562  * This must be called from context that can sleep.
 563  * Return: the crypto engine structure on success, else NULL.
 564  */
 565 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
 566 {
 567         return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
 568                                                 CRYPTO_ENGINE_MAX_QLEN);
 569 }
 570 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
 571
 572 /**
 573  * crypto_engine_exit - free the resources of hardware engine when exit
 574  * @engine: the hardware engine need to be freed
 575  *
 576  * Return 0 for success.
 577  */
 578 int crypto_engine_exit(struct crypto_engine *engine)
 579 {
 580         int ret;
 581
 582         ret = crypto_engine_stop(engine);
 583         if (ret)
 584                 return ret;
 585
 586         kthread_destroy_worker(engine->kworker);
 587
 588         return 0;
 589 }
 590 EXPORT_SYMBOL_GPL(crypto_engine_exit);
 591
 592 MODULE_LICENSE("GPL");
 593 MODULE_DESCRIPTION("Crypto hardware engine framework");