drivers/crypto/omap-aes.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for OMAP AES HW acceleration.
   5  *
   6  * Copyright (c) 2010 Nokia Corporation
   7  * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as published
  11  * by the Free Software Foundation.
  12  *
  13  */
  14
  15 #define pr_fmt(fmt) "%s: " fmt, __func__
  16
  17 #include <linux/err.h>
  18 #include <linux/module.h>
  19 #include <linux/init.h>
  20 #include <linux/errno.h>
  21 #include <linux/kernel.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/pm_runtime.h>
  26 #include <linux/io.h>
  27 #include <linux/crypto.h>
  28 #include <linux/interrupt.h>
  29 #include <crypto/scatterwalk.h>
  30 #include <crypto/aes.h>
  31
  32 #include <linux/omap-dma.h>
  33
  34 /* OMAP TRM gives bitfields as start:end, where start is the higher bit
  35    number. For example 7:0 */
  36 #define FLD_MASK(start, end)    (((1 << ((start) - (end) + 1)) - 1) << (end))
  37 #define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
  38
  39 #define AES_REG_KEY(x)                  (0x1C - ((x ^ 0x01) * 0x04))
  40 #define AES_REG_IV(x)                   (0x20 + ((x) * 0x04))
  41
  42 #define AES_REG_CTRL                    0x30
  43 #define AES_REG_CTRL_CTR_WIDTH          (1 << 7)
  44 #define AES_REG_CTRL_CTR                (1 << 6)
  45 #define AES_REG_CTRL_CBC                (1 << 5)
  46 #define AES_REG_CTRL_KEY_SIZE           (3 << 3)
  47 #define AES_REG_CTRL_DIRECTION          (1 << 2)
  48 #define AES_REG_CTRL_INPUT_READY        (1 << 1)
  49 #define AES_REG_CTRL_OUTPUT_READY       (1 << 0)
  50
  51 #define AES_REG_DATA                    0x34
  52 #define AES_REG_DATA_N(x)               (0x34 + ((x) * 0x04))
  53
  54 #define AES_REG_REV                     0x44
  55 #define AES_REG_REV_MAJOR               0xF0
  56 #define AES_REG_REV_MINOR               0x0F
  57
  58 #define AES_REG_MASK                    0x48
  59 #define AES_REG_MASK_SIDLE              (1 << 6)
  60 #define AES_REG_MASK_START              (1 << 5)
  61 #define AES_REG_MASK_DMA_OUT_EN         (1 << 3)
  62 #define AES_REG_MASK_DMA_IN_EN          (1 << 2)
  63 #define AES_REG_MASK_SOFTRESET          (1 << 1)
  64 #define AES_REG_AUTOIDLE                (1 << 0)
  65
  66 #define AES_REG_SYSSTATUS               0x4C
  67 #define AES_REG_SYSSTATUS_RESETDONE     (1 << 0)
  68
  69 #define DEFAULT_TIMEOUT         (5*HZ)
  70
  71 #define FLAGS_MODE_MASK         0x000f
  72 #define FLAGS_ENCRYPT           BIT(0)
  73 #define FLAGS_CBC               BIT(1)
  74 #define FLAGS_GIV               BIT(2)
  75
  76 #define FLAGS_INIT              BIT(4)
  77 #define FLAGS_FAST              BIT(5)
  78 #define FLAGS_BUSY              BIT(6)
  79
  80 struct omap_aes_ctx {
  81         struct omap_aes_dev *dd;
  82
  83         int             keylen;
  84         u32             key[AES_KEYSIZE_256 / sizeof(u32)];
  85         unsigned long   flags;
  86 };
  87
  88 struct omap_aes_reqctx {
  89         unsigned long mode;
  90 };
  91
  92 #define OMAP_AES_QUEUE_LENGTH   1
  93 #define OMAP_AES_CACHE_SIZE     0
  94
  95 struct omap_aes_dev {
  96         struct list_head        list;
  97         unsigned long           phys_base;
  98         void __iomem            *io_base;
  99         struct omap_aes_ctx     *ctx;
 100         struct device           *dev;
 101         unsigned long           flags;
 102         int                     err;
 103
 104         spinlock_t              lock;
 105         struct crypto_queue     queue;
 106
 107         struct tasklet_struct   done_task;
 108         struct tasklet_struct   queue_task;
 109
 110         struct ablkcipher_request       *req;
 111         size_t                          total;
 112         struct scatterlist              *in_sg;
 113         size_t                          in_offset;
 114         struct scatterlist              *out_sg;
 115         size_t                          out_offset;
 116
 117         size_t                  buflen;
 118         void                    *buf_in;
 119         size_t                  dma_size;
 120         int                     dma_in;
 121         int                     dma_lch_in;
 122         dma_addr_t              dma_addr_in;
 123         void                    *buf_out;
 124         int                     dma_out;
 125         int                     dma_lch_out;
 126         dma_addr_t              dma_addr_out;
 127 };
 128
 129 /* keep registered devices data here */
 130 static LIST_HEAD(dev_list);
 131 static DEFINE_SPINLOCK(list_lock);
 132
 133 static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
 134 {
 135         return __raw_readl(dd->io_base + offset);
 136 }
 137
 138 static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
 139                                   u32 value)
 140 {
 141         __raw_writel(value, dd->io_base + offset);
 142 }
 143
 144 static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
 145                                         u32 value, u32 mask)
 146 {
 147         u32 val;
 148
 149         val = omap_aes_read(dd, offset);
 150         val &= ~mask;
 151         val |= value;
 152         omap_aes_write(dd, offset, val);
 153 }
 154
 155 static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
 156                                         u32 *value, int count)
 157 {
 158         for (; count--; value++, offset += 4)
 159                 omap_aes_write(dd, offset, *value);
 160 }
 161
 162 static int omap_aes_hw_init(struct omap_aes_dev *dd)
 163 {
 164         /*
 165          * clocks are enabled when request starts and disabled when finished.
 166          * It may be long delays between requests.
 167          * Device might go to off mode to save power.
 168          */
 169         pm_runtime_get_sync(dd->dev);
 170
 171         if (!(dd->flags & FLAGS_INIT)) {
 172                 dd->flags |= FLAGS_INIT;
 173                 dd->err = 0;
 174         }
 175
 176         return 0;
 177 }
 178
 179 static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
 180 {
 181         unsigned int key32;
 182         int i, err;
 183         u32 val, mask;
 184
 185         err = omap_aes_hw_init(dd);
 186         if (err)
 187                 return err;
 188
 189         val = 0;
 190         if (dd->dma_lch_out >= 0)
 191                 val |= AES_REG_MASK_DMA_OUT_EN;
 192         if (dd->dma_lch_in >= 0)
 193                 val |= AES_REG_MASK_DMA_IN_EN;
 194
 195         mask = AES_REG_MASK_DMA_IN_EN | AES_REG_MASK_DMA_OUT_EN;
 196
 197         omap_aes_write_mask(dd, AES_REG_MASK, val, mask);
 198
 199         key32 = dd->ctx->keylen / sizeof(u32);
 200
 201         /* it seems a key should always be set even if it has not changed */
 202         for (i = 0; i < key32; i++) {
 203                 omap_aes_write(dd, AES_REG_KEY(i),
 204                         __le32_to_cpu(dd->ctx->key[i]));
 205         }
 206
 207         if ((dd->flags & FLAGS_CBC) && dd->req->info)
 208                 omap_aes_write_n(dd, AES_REG_IV(0), dd->req->info, 4);
 209
 210         val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
 211         if (dd->flags & FLAGS_CBC)
 212                 val |= AES_REG_CTRL_CBC;
 213         if (dd->flags & FLAGS_ENCRYPT)
 214                 val |= AES_REG_CTRL_DIRECTION;
 215
 216         mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION |
 217                         AES_REG_CTRL_KEY_SIZE;
 218
 219         omap_aes_write_mask(dd, AES_REG_CTRL, val, mask);
 220
 221         /* IN */
 222         omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT,
 223                                  dd->phys_base + AES_REG_DATA, 0, 4);
 224
 225         omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 226         omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 227
 228         /* OUT */
 229         omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
 230                                 dd->phys_base + AES_REG_DATA, 0, 4);
 231
 232         omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 233         omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 234
 235         return 0;
 236 }
 237
 238 static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
 239 {
 240         struct omap_aes_dev *dd = NULL, *tmp;
 241
 242         spin_lock_bh(&list_lock);
 243         if (!ctx->dd) {
 244                 list_for_each_entry(tmp, &dev_list, list) {
 245                         /* FIXME: take fist available aes core */
 246                         dd = tmp;
 247                         break;
 248                 }
 249                 ctx->dd = dd;
 250         } else {
 251                 /* already found before */
 252                 dd = ctx->dd;
 253         }
 254         spin_unlock_bh(&list_lock);
 255
 256         return dd;
 257 }
 258
 259 static void omap_aes_dma_callback(int lch, u16 ch_status, void *data)
 260 {
 261         struct omap_aes_dev *dd = data;
 262
 263         if (ch_status != OMAP_DMA_BLOCK_IRQ) {
 264                 pr_err("omap-aes DMA error status: 0x%hx\n", ch_status);
 265                 dd->err = -EIO;
 266                 dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
 267         } else if (lch == dd->dma_lch_in) {
 268                 return;
 269         }
 270
 271         /* dma_lch_out - completed */
 272         tasklet_schedule(&dd->done_task);
 273 }
 274
 275 static int omap_aes_dma_init(struct omap_aes_dev *dd)
 276 {
 277         int err = -ENOMEM;
 278
 279         dd->dma_lch_out = -1;
 280         dd->dma_lch_in = -1;
 281
 282         dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 283         dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 284         dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE;
 285         dd->buflen &= ~(AES_BLOCK_SIZE - 1);
 286
 287         if (!dd->buf_in || !dd->buf_out) {
 288                 dev_err(dd->dev, "unable to alloc pages.\n");
 289                 goto err_alloc;
 290         }
 291
 292         /* MAP here */
 293         dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen,
 294                                          DMA_TO_DEVICE);
 295         if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
 296                 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 297                 err = -EINVAL;
 298                 goto err_map_in;
 299         }
 300
 301         dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen,
 302                                           DMA_FROM_DEVICE);
 303         if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
 304                 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 305                 err = -EINVAL;
 306                 goto err_map_out;
 307         }
 308
 309         err = omap_request_dma(dd->dma_in, "omap-aes-rx",
 310                                omap_aes_dma_callback, dd, &dd->dma_lch_in);
 311         if (err) {
 312                 dev_err(dd->dev, "Unable to request DMA channel\n");
 313                 goto err_dma_in;
 314         }
 315         err = omap_request_dma(dd->dma_out, "omap-aes-tx",
 316                                omap_aes_dma_callback, dd, &dd->dma_lch_out);
 317         if (err) {
 318                 dev_err(dd->dev, "Unable to request DMA channel\n");
 319                 goto err_dma_out;
 320         }
 321
 322         return 0;
 323
 324 err_dma_out:
 325         omap_free_dma(dd->dma_lch_in);
 326 err_dma_in:
 327         dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 328                          DMA_FROM_DEVICE);
 329 err_map_out:
 330         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 331 err_map_in:
 332         free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 333         free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 334 err_alloc:
 335         if (err)
 336                 pr_err("error: %d\n", err);
 337         return err;
 338 }
 339
 340 static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
 341 {
 342         omap_free_dma(dd->dma_lch_out);
 343         omap_free_dma(dd->dma_lch_in);
 344         dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 345                          DMA_FROM_DEVICE);
 346         dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 347         free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 348         free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 349 }
 350
 351 static void sg_copy_buf(void *buf, struct scatterlist *sg,
 352                               unsigned int start, unsigned int nbytes, int out)
 353 {
 354         struct scatter_walk walk;
 355
 356         if (!nbytes)
 357                 return;
 358
 359         scatterwalk_start(&walk, sg);
 360         scatterwalk_advance(&walk, start);
 361         scatterwalk_copychunks(buf, &walk, nbytes, out);
 362         scatterwalk_done(&walk, out, 0);
 363 }
 364
 365 static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
 366                    size_t buflen, size_t total, int out)
 367 {
 368         unsigned int count, off = 0;
 369
 370         while (buflen && total) {
 371                 count = min((*sg)->length - *offset, total);
 372                 count = min(count, buflen);
 373
 374                 if (!count)
 375                         return off;
 376
 377                 /*
 378                  * buflen and total are AES_BLOCK_SIZE size aligned,
 379                  * so count should be also aligned
 380                  */
 381
 382                 sg_copy_buf(buf + off, *sg, *offset, count, out);
 383
 384                 off += count;
 385                 buflen -= count;
 386                 *offset += count;
 387                 total -= count;
 388
 389                 if (*offset == (*sg)->length) {
 390                         *sg = sg_next(*sg);
 391                         if (*sg)
 392                                 *offset = 0;
 393                         else
 394                                 total = 0;
 395                 }
 396         }
 397
 398         return off;
 399 }
 400
 401 static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 402                                dma_addr_t dma_addr_out, int length)
 403 {
 404         struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 405         struct omap_aes_dev *dd = ctx->dd;
 406         int len32;
 407
 408         pr_debug("len: %d\n", length);
 409
 410         dd->dma_size = length;
 411
 412         if (!(dd->flags & FLAGS_FAST))
 413                 dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 414                                            DMA_TO_DEVICE);
 415
 416         len32 = DIV_ROUND_UP(length, sizeof(u32));
 417
 418         /* IN */
 419         omap_set_dma_transfer_params(dd->dma_lch_in, OMAP_DMA_DATA_TYPE_S32,
 420                                      len32, 1, OMAP_DMA_SYNC_PACKET, dd->dma_in,
 421                                         OMAP_DMA_DST_SYNC);
 422
 423         omap_set_dma_src_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_POST_INC,
 424                                 dma_addr_in, 0, 0);
 425
 426         /* OUT */
 427         omap_set_dma_transfer_params(dd->dma_lch_out, OMAP_DMA_DATA_TYPE_S32,
 428                                      len32, 1, OMAP_DMA_SYNC_PACKET,
 429                                         dd->dma_out, OMAP_DMA_SRC_SYNC);
 430
 431         omap_set_dma_dest_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_POST_INC,
 432                                  dma_addr_out, 0, 0);
 433
 434         omap_start_dma(dd->dma_lch_in);
 435         omap_start_dma(dd->dma_lch_out);
 436
 437         /* start DMA or disable idle mode */
 438         omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START,
 439                             AES_REG_MASK_START);
 440
 441         return 0;
 442 }
 443
 444 static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
 445 {
 446         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 447                                         crypto_ablkcipher_reqtfm(dd->req));
 448         int err, fast = 0, in, out;
 449         size_t count;
 450         dma_addr_t addr_in, addr_out;
 451
 452         pr_debug("total: %d\n", dd->total);
 453
 454         if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) {
 455                 /* check for alignment */
 456                 in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32));
 457                 out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32));
 458
 459                 fast = in && out;
 460         }
 461
 462         if (fast)  {
 463                 count = min(dd->total, sg_dma_len(dd->in_sg));
 464                 count = min(count, sg_dma_len(dd->out_sg));
 465
 466                 if (count != dd->total) {
 467                         pr_err("request length != buffer length\n");
 468                         return -EINVAL;
 469                 }
 470
 471                 pr_debug("fast\n");
 472
 473                 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 474                 if (!err) {
 475                         dev_err(dd->dev, "dma_map_sg() error\n");
 476                         return -EINVAL;
 477                 }
 478
 479                 err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 480                 if (!err) {
 481                         dev_err(dd->dev, "dma_map_sg() error\n");
 482                         dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 483                         return -EINVAL;
 484                 }
 485
 486                 addr_in = sg_dma_address(dd->in_sg);
 487                 addr_out = sg_dma_address(dd->out_sg);
 488
 489                 dd->flags |= FLAGS_FAST;
 490
 491         } else {
 492                 /* use cache buffers */
 493                 count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
 494                                  dd->buflen, dd->total, 0);
 495
 496                 addr_in = dd->dma_addr_in;
 497                 addr_out = dd->dma_addr_out;
 498
 499                 dd->flags &= ~FLAGS_FAST;
 500
 501         }
 502
 503         dd->total -= count;
 504
 505         err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count);
 506         if (err) {
 507                 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 508                 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
 509         }
 510
 511         return err;
 512 }
 513
 514 static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
 515 {
 516         struct ablkcipher_request *req = dd->req;
 517
 518         pr_debug("err: %d\n", err);
 519
 520         pm_runtime_put_sync(dd->dev);
 521         dd->flags &= ~FLAGS_BUSY;
 522
 523         req->base.complete(&req->base, err);
 524 }
 525
 526 static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
 527 {
 528         int err = 0;
 529         size_t count;
 530
 531         pr_debug("total: %d\n", dd->total);
 532
 533         omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START);
 534
 535         omap_stop_dma(dd->dma_lch_in);
 536         omap_stop_dma(dd->dma_lch_out);
 537
 538         if (dd->flags & FLAGS_FAST) {
 539                 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 540                 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 541         } else {
 542                 dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 543                                            dd->dma_size, DMA_FROM_DEVICE);
 544
 545                 /* copy data */
 546                 count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out,
 547                                  dd->buflen, dd->dma_size, 1);
 548                 if (count != dd->dma_size) {
 549                         err = -EINVAL;
 550                         pr_err("not all data converted: %u\n", count);
 551                 }
 552         }
 553
 554         return err;
 555 }
 556
 557 static int omap_aes_handle_queue(struct omap_aes_dev *dd,
 558                                struct ablkcipher_request *req)
 559 {
 560         struct crypto_async_request *async_req, *backlog;
 561         struct omap_aes_ctx *ctx;
 562         struct omap_aes_reqctx *rctx;
 563         unsigned long flags;
 564         int err, ret = 0;
 565
 566         spin_lock_irqsave(&dd->lock, flags);
 567         if (req)
 568                 ret = ablkcipher_enqueue_request(&dd->queue, req);
 569         if (dd->flags & FLAGS_BUSY) {
 570                 spin_unlock_irqrestore(&dd->lock, flags);
 571                 return ret;
 572         }
 573         backlog = crypto_get_backlog(&dd->queue);
 574         async_req = crypto_dequeue_request(&dd->queue);
 575         if (async_req)
 576                 dd->flags |= FLAGS_BUSY;
 577         spin_unlock_irqrestore(&dd->lock, flags);
 578
 579         if (!async_req)
 580                 return ret;
 581
 582         if (backlog)
 583                 backlog->complete(backlog, -EINPROGRESS);
 584
 585         req = ablkcipher_request_cast(async_req);
 586
 587         /* assign new request to device */
 588         dd->req = req;
 589         dd->total = req->nbytes;
 590         dd->in_offset = 0;
 591         dd->in_sg = req->src;
 592         dd->out_offset = 0;
 593         dd->out_sg = req->dst;
 594
 595         rctx = ablkcipher_request_ctx(req);
 596         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 597         rctx->mode &= FLAGS_MODE_MASK;
 598         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 599
 600         dd->ctx = ctx;
 601         ctx->dd = dd;
 602
 603         err = omap_aes_write_ctrl(dd);
 604         if (!err)
 605                 err = omap_aes_crypt_dma_start(dd);
 606         if (err) {
 607                 /* aes_task will not finish it, so do it here */
 608                 omap_aes_finish_req(dd, err);
 609                 tasklet_schedule(&dd->queue_task);
 610         }
 611
 612         return ret; /* return ret, which is enqueue return value */
 613 }
 614
 615 static void omap_aes_done_task(unsigned long data)
 616 {
 617         struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 618         int err;
 619
 620         pr_debug("enter\n");
 621
 622         err = omap_aes_crypt_dma_stop(dd);
 623
 624         err = dd->err ? : err;
 625
 626         if (dd->total && !err) {
 627                 err = omap_aes_crypt_dma_start(dd);
 628                 if (!err)
 629                         return; /* DMA started. Not fininishing. */
 630         }
 631
 632         omap_aes_finish_req(dd, err);
 633         omap_aes_handle_queue(dd, NULL);
 634
 635         pr_debug("exit\n");
 636 }
 637
 638 static void omap_aes_queue_task(unsigned long data)
 639 {
 640         struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 641
 642         omap_aes_handle_queue(dd, NULL);
 643 }
 644
 645 static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 646 {
 647         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 648                         crypto_ablkcipher_reqtfm(req));
 649         struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 650         struct omap_aes_dev *dd;
 651
 652         pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 653                   !!(mode & FLAGS_ENCRYPT),
 654                   !!(mode & FLAGS_CBC));
 655
 656         if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
 657                 pr_err("request size is not exact amount of AES blocks\n");
 658                 return -EINVAL;
 659         }
 660
 661         dd = omap_aes_find_dev(ctx);
 662         if (!dd)
 663                 return -ENODEV;
 664
 665         rctx->mode = mode;
 666
 667         return omap_aes_handle_queue(dd, req);
 668 }
 669
 670 /* ********************** ALG API ************************************ */
 671
 672 static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 673                            unsigned int keylen)
 674 {
 675         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 676
 677         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 678                    keylen != AES_KEYSIZE_256)
 679                 return -EINVAL;
 680
 681         pr_debug("enter, keylen: %d\n", keylen);
 682
 683         memcpy(ctx->key, key, keylen);
 684         ctx->keylen = keylen;
 685
 686         return 0;
 687 }
 688
 689 static int omap_aes_ecb_encrypt(struct ablkcipher_request *req)
 690 {
 691         return omap_aes_crypt(req, FLAGS_ENCRYPT);
 692 }
 693
 694 static int omap_aes_ecb_decrypt(struct ablkcipher_request *req)
 695 {
 696         return omap_aes_crypt(req, 0);
 697 }
 698
 699 static int omap_aes_cbc_encrypt(struct ablkcipher_request *req)
 700 {
 701         return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 702 }
 703
 704 static int omap_aes_cbc_decrypt(struct ablkcipher_request *req)
 705 {
 706         return omap_aes_crypt(req, FLAGS_CBC);
 707 }
 708
 709 static int omap_aes_cra_init(struct crypto_tfm *tfm)
 710 {
 711         pr_debug("enter\n");
 712
 713         tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx);
 714
 715         return 0;
 716 }
 717
 718 static void omap_aes_cra_exit(struct crypto_tfm *tfm)
 719 {
 720         pr_debug("enter\n");
 721 }
 722
 723 /* ********************** ALGS ************************************ */
 724
 725 static struct crypto_alg algs[] = {
 726 {
 727         .cra_name               = "ecb(aes)",
 728         .cra_driver_name        = "ecb-aes-omap",
 729         .cra_priority           = 100,
 730         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 731                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 732                                   CRYPTO_ALG_ASYNC,
 733         .cra_blocksize          = AES_BLOCK_SIZE,
 734         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 735         .cra_alignmask          = 0,
 736         .cra_type               = &crypto_ablkcipher_type,
 737         .cra_module             = THIS_MODULE,
 738         .cra_init               = omap_aes_cra_init,
 739         .cra_exit               = omap_aes_cra_exit,
 740         .cra_u.ablkcipher = {
 741                 .min_keysize    = AES_MIN_KEY_SIZE,
 742                 .max_keysize    = AES_MAX_KEY_SIZE,
 743                 .setkey         = omap_aes_setkey,
 744                 .encrypt        = omap_aes_ecb_encrypt,
 745                 .decrypt        = omap_aes_ecb_decrypt,
 746         }
 747 },
 748 {
 749         .cra_name               = "cbc(aes)",
 750         .cra_driver_name        = "cbc-aes-omap",
 751         .cra_priority           = 100,
 752         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 753                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 754                                   CRYPTO_ALG_ASYNC,
 755         .cra_blocksize          = AES_BLOCK_SIZE,
 756         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 757         .cra_alignmask          = 0,
 758         .cra_type               = &crypto_ablkcipher_type,
 759         .cra_module             = THIS_MODULE,
 760         .cra_init               = omap_aes_cra_init,
 761         .cra_exit               = omap_aes_cra_exit,
 762         .cra_u.ablkcipher = {
 763                 .min_keysize    = AES_MIN_KEY_SIZE,
 764                 .max_keysize    = AES_MAX_KEY_SIZE,
 765                 .ivsize         = AES_BLOCK_SIZE,
 766                 .setkey         = omap_aes_setkey,
 767                 .encrypt        = omap_aes_cbc_encrypt,
 768                 .decrypt        = omap_aes_cbc_decrypt,
 769         }
 770 }
 771 };
 772
 773 static int omap_aes_probe(struct platform_device *pdev)
 774 {
 775         struct device *dev = &pdev->dev;
 776         struct omap_aes_dev *dd;
 777         struct resource *res;
 778         int err = -ENOMEM, i, j;
 779         u32 reg;
 780
 781         dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL);
 782         if (dd == NULL) {
 783                 dev_err(dev, "unable to alloc data struct.\n");
 784                 goto err_data;
 785         }
 786         dd->dev = dev;
 787         platform_set_drvdata(pdev, dd);
 788
 789         spin_lock_init(&dd->lock);
 790         crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH);
 791
 792         /* Get the base address */
 793         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 794         if (!res) {
 795                 dev_err(dev, "invalid resource type\n");
 796                 err = -ENODEV;
 797                 goto err_res;
 798         }
 799         dd->phys_base = res->start;
 800
 801         /* Get the DMA */
 802         res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
 803         if (!res)
 804                 dev_info(dev, "no DMA info\n");
 805         else
 806                 dd->dma_out = res->start;
 807
 808         /* Get the DMA */
 809         res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
 810         if (!res)
 811                 dev_info(dev, "no DMA info\n");
 812         else
 813                 dd->dma_in = res->start;
 814
 815         dd->io_base = ioremap(dd->phys_base, SZ_4K);
 816         if (!dd->io_base) {
 817                 dev_err(dev, "can't ioremap\n");
 818                 err = -ENOMEM;
 819                 goto err_res;
 820         }
 821
 822         pm_runtime_enable(dev);
 823         pm_runtime_get_sync(dev);
 824
 825         reg = omap_aes_read(dd, AES_REG_REV);
 826         dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
 827                  (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR);
 828
 829         pm_runtime_put_sync(dev);
 830
 831         tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
 832         tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
 833
 834         err = omap_aes_dma_init(dd);
 835         if (err)
 836                 goto err_dma;
 837
 838         INIT_LIST_HEAD(&dd->list);
 839         spin_lock(&list_lock);
 840         list_add_tail(&dd->list, &dev_list);
 841         spin_unlock(&list_lock);
 842
 843         for (i = 0; i < ARRAY_SIZE(algs); i++) {
 844                 pr_debug("i: %d\n", i);
 845                 err = crypto_register_alg(&algs[i]);
 846                 if (err)
 847                         goto err_algs;
 848         }
 849
 850         return 0;
 851 err_algs:
 852         for (j = 0; j < i; j++)
 853                 crypto_unregister_alg(&algs[j]);
 854         omap_aes_dma_cleanup(dd);
 855 err_dma:
 856         tasklet_kill(&dd->done_task);
 857         tasklet_kill(&dd->queue_task);
 858         iounmap(dd->io_base);
 859         pm_runtime_disable(dev);
 860 err_res:
 861         kfree(dd);
 862         dd = NULL;
 863 err_data:
 864         dev_err(dev, "initialization failed.\n");
 865         return err;
 866 }
 867
 868 static int omap_aes_remove(struct platform_device *pdev)
 869 {
 870         struct omap_aes_dev *dd = platform_get_drvdata(pdev);
 871         int i;
 872
 873         if (!dd)
 874                 return -ENODEV;
 875
 876         spin_lock(&list_lock);
 877         list_del(&dd->list);
 878         spin_unlock(&list_lock);
 879
 880         for (i = 0; i < ARRAY_SIZE(algs); i++)
 881                 crypto_unregister_alg(&algs[i]);
 882
 883         tasklet_kill(&dd->done_task);
 884         tasklet_kill(&dd->queue_task);
 885         omap_aes_dma_cleanup(dd);
 886         iounmap(dd->io_base);
 887         pm_runtime_disable(dd->dev);
 888         kfree(dd);
 889         dd = NULL;
 890
 891         return 0;
 892 }
 893
 894 #ifdef CONFIG_PM_SLEEP
 895 static int omap_aes_suspend(struct device *dev)
 896 {
 897         pm_runtime_put_sync(dev);
 898         return 0;
 899 }
 900
 901 static int omap_aes_resume(struct device *dev)
 902 {
 903         pm_runtime_get_sync(dev);
 904         return 0;
 905 }
 906 #endif
 907
 908 static const struct dev_pm_ops omap_aes_pm_ops = {
 909         SET_SYSTEM_SLEEP_PM_OPS(omap_aes_suspend, omap_aes_resume)
 910 };
 911
 912 static struct platform_driver omap_aes_driver = {
 913         .probe  = omap_aes_probe,
 914         .remove = omap_aes_remove,
 915         .driver = {
 916                 .name   = "omap-aes",
 917                 .owner  = THIS_MODULE,
 918                 .pm     = &omap_aes_pm_ops,
 919         },
 920 };
 921
 922 static int __init omap_aes_mod_init(void)
 923 {
 924         return  platform_driver_register(&omap_aes_driver);
 925 }
 926
 927 static void __exit omap_aes_mod_exit(void)
 928 {
 929         platform_driver_unregister(&omap_aes_driver);
 930 }
 931
 932 module_init(omap_aes_mod_init);
 933 module_exit(omap_aes_mod_exit);
 934
 935 MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
 936 MODULE_LICENSE("GPL v2");
 937 MODULE_AUTHOR("Dmitry Kasatkin");
 938