drivers/crypto/ux500/cryp/cryp_core.c

   1 /**
   2  * Copyright (C) ST-Ericsson SA 2010
   3  * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson.
   4  * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson.
   5  * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson.
   6  * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson.
   7  * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson.
   8  * Author: Andreas Westin <andreas.westin@stericsson.com> for ST-Ericsson.
   9  * License terms: GNU General Public License (GPL) version 2
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/completion.h>
  14 #include <linux/crypto.h>
  15 #include <linux/dmaengine.h>
  16 #include <linux/err.h>
  17 #include <linux/errno.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/io.h>
  20 #include <linux/irqreturn.h>
  21 #include <linux/klist.h>
  22 #include <linux/module.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/regulator/consumer.h>
  25 #include <linux/semaphore.h>
  26 #include <linux/platform_data/dma-ste-dma40.h>
  27
  28 #include <crypto/aes.h>
  29 #include <crypto/algapi.h>
  30 #include <crypto/ctr.h>
  31 #include <crypto/des.h>
  32 #include <crypto/scatterwalk.h>
  33
  34 #include <linux/platform_data/crypto-ux500.h>
  35
  36 #include "cryp_p.h"
  37 #include "cryp.h"
  38
  39 #define CRYP_MAX_KEY_SIZE       32
  40 #define BYTES_PER_WORD          4
  41
  42 static int cryp_mode;
  43 static atomic_t session_id;
  44
  45 static struct stedma40_chan_cfg *mem_to_engine;
  46 static struct stedma40_chan_cfg *engine_to_mem;
  47
  48 /**
  49  * struct cryp_driver_data - data specific to the driver.
  50  *
  51  * @device_list: A list of registered devices to choose from.
  52  * @device_allocation: A semaphore initialized with number of devices.
  53  */
  54 struct cryp_driver_data {
  55         struct klist device_list;
  56         struct semaphore device_allocation;
  57 };
  58
  59 /**
  60  * struct cryp_ctx - Crypto context
  61  * @config: Crypto mode.
  62  * @key[CRYP_MAX_KEY_SIZE]: Key.
  63  * @keylen: Length of key.
  64  * @iv: Pointer to initialization vector.
  65  * @indata: Pointer to indata.
  66  * @outdata: Pointer to outdata.
  67  * @datalen: Length of indata.
  68  * @outlen: Length of outdata.
  69  * @blocksize: Size of blocks.
  70  * @updated: Updated flag.
  71  * @dev_ctx: Device dependent context.
  72  * @device: Pointer to the device.
  73  */
  74 struct cryp_ctx {
  75         struct cryp_config config;
  76         u8 key[CRYP_MAX_KEY_SIZE];
  77         u32 keylen;
  78         u8 *iv;
  79         const u8 *indata;
  80         u8 *outdata;
  81         u32 datalen;
  82         u32 outlen;
  83         u32 blocksize;
  84         u8 updated;
  85         struct cryp_device_context dev_ctx;
  86         struct cryp_device_data *device;
  87         u32 session_id;
  88 };
  89
  90 static struct cryp_driver_data driver_data;
  91
  92 /**
  93  * uint8p_to_uint32_be - 4*uint8 to uint32 big endian
  94  * @in: Data to convert.
  95  */
  96 static inline u32 uint8p_to_uint32_be(u8 *in)
  97 {
  98         u32 *data = (u32 *)in;
  99
 100         return cpu_to_be32p(data);
 101 }
 102
 103 /**
 104  * swap_bits_in_byte - mirror the bits in a byte
 105  * @b: the byte to be mirrored
 106  *
 107  * The bits are swapped the following way:
 108  *  Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and
 109  *  nibble 2 (n2) bits 4-7.
 110  *
 111  *  Nibble 1 (n1):
 112  *  (The "old" (moved) bit is replaced with a zero)
 113  *  1. Move bit 6 and 7, 4 positions to the left.
 114  *  2. Move bit 3 and 5, 2 positions to the left.
 115  *  3. Move bit 1-4, 1 position to the left.
 116  *
 117  *  Nibble 2 (n2):
 118  *  1. Move bit 0 and 1, 4 positions to the right.
 119  *  2. Move bit 2 and 4, 2 positions to the right.
 120  *  3. Move bit 3-6, 1 position to the right.
 121  *
 122  *  Combine the two nibbles to a complete and swapped byte.
 123  */
 124
 125 static inline u8 swap_bits_in_byte(u8 b)
 126 {
 127 #define R_SHIFT_4_MASK  0xc0 /* Bits 6 and 7, right shift 4 */
 128 #define R_SHIFT_2_MASK  0x28 /* (After right shift 4) Bits 3 and 5,
 129                                   right shift 2 */
 130 #define R_SHIFT_1_MASK  0x1e /* (After right shift 2) Bits 1-4,
 131                                   right shift 1 */
 132 #define L_SHIFT_4_MASK  0x03 /* Bits 0 and 1, left shift 4 */
 133 #define L_SHIFT_2_MASK  0x14 /* (After left shift 4) Bits 2 and 4,
 134                                   left shift 2 */
 135 #define L_SHIFT_1_MASK  0x78 /* (After left shift 1) Bits 3-6,
 136                                   left shift 1 */
 137
 138         u8 n1;
 139         u8 n2;
 140
 141         /* Swap most significant nibble */
 142         /* Right shift 4, bits 6 and 7 */
 143         n1 = ((b  & R_SHIFT_4_MASK) >> 4) | (b  & ~(R_SHIFT_4_MASK >> 4));
 144         /* Right shift 2, bits 3 and 5 */
 145         n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2));
 146         /* Right shift 1, bits 1-4 */
 147         n1 = (n1  & R_SHIFT_1_MASK) >> 1;
 148
 149         /* Swap least significant nibble */
 150         /* Left shift 4, bits 0 and 1 */
 151         n2 = ((b  & L_SHIFT_4_MASK) << 4) | (b  & ~(L_SHIFT_4_MASK << 4));
 152         /* Left shift 2, bits 2 and 4 */
 153         n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2));
 154         /* Left shift 1, bits 3-6 */
 155         n2 = (n2  & L_SHIFT_1_MASK) << 1;
 156
 157         return n1 | n2;
 158 }
 159
 160 static inline void swap_words_in_key_and_bits_in_byte(const u8 *in,
 161                                                       u8 *out, u32 len)
 162 {
 163         unsigned int i = 0;
 164         int j;
 165         int index = 0;
 166
 167         j = len - BYTES_PER_WORD;
 168         while (j >= 0) {
 169                 for (i = 0; i < BYTES_PER_WORD; i++) {
 170                         index = len - j - BYTES_PER_WORD + i;
 171                         out[j + i] =
 172                                 swap_bits_in_byte(in[index]);
 173                 }
 174                 j -= BYTES_PER_WORD;
 175         }
 176 }
 177
 178 static void add_session_id(struct cryp_ctx *ctx)
 179 {
 180         /*
 181          * We never want 0 to be a valid value, since this is the default value
 182          * for the software context.
 183          */
 184         if (unlikely(atomic_inc_and_test(&session_id)))
 185                 atomic_inc(&session_id);
 186
 187         ctx->session_id = atomic_read(&session_id);
 188 }
 189
 190 static irqreturn_t cryp_interrupt_handler(int irq, void *param)
 191 {
 192         struct cryp_ctx *ctx;
 193         int i;
 194         struct cryp_device_data *device_data;
 195
 196         if (param == NULL) {
 197                 BUG_ON(!param);
 198                 return IRQ_HANDLED;
 199         }
 200
 201         /* The device is coming from the one found in hw_crypt_noxts. */
 202         device_data = (struct cryp_device_data *)param;
 203
 204         ctx = device_data->current_ctx;
 205
 206         if (ctx == NULL) {
 207                 BUG_ON(!ctx);
 208                 return IRQ_HANDLED;
 209         }
 210
 211         dev_dbg(ctx->device->dev, "[%s] (len: %d) %s, ", __func__, ctx->outlen,
 212                 cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO) ?
 213                 "out" : "in");
 214
 215         if (cryp_pending_irq_src(device_data,
 216                                  CRYP_IRQ_SRC_OUTPUT_FIFO)) {
 217                 if (ctx->outlen / ctx->blocksize > 0) {
 218                         for (i = 0; i < ctx->blocksize / 4; i++) {
 219                                 *(ctx->outdata) = readl_relaxed(
 220                                                 &device_data->base->dout);
 221                                 ctx->outdata += 4;
 222                                 ctx->outlen -= 4;
 223                         }
 224
 225                         if (ctx->outlen == 0) {
 226                                 cryp_disable_irq_src(device_data,
 227                                                      CRYP_IRQ_SRC_OUTPUT_FIFO);
 228                         }
 229                 }
 230         } else if (cryp_pending_irq_src(device_data,
 231                                         CRYP_IRQ_SRC_INPUT_FIFO)) {
 232                 if (ctx->datalen / ctx->blocksize > 0) {
 233                         for (i = 0 ; i < ctx->blocksize / 4; i++) {
 234                                 writel_relaxed(ctx->indata,
 235                                                 &device_data->base->din);
 236                                 ctx->indata += 4;
 237                                 ctx->datalen -= 4;
 238                         }
 239
 240                         if (ctx->datalen == 0)
 241                                 cryp_disable_irq_src(device_data,
 242                                                    CRYP_IRQ_SRC_INPUT_FIFO);
 243
 244                         if (ctx->config.algomode == CRYP_ALGO_AES_XTS) {
 245                                 CRYP_PUT_BITS(&device_data->base->cr,
 246                                               CRYP_START_ENABLE,
 247                                               CRYP_CR_START_POS,
 248                                               CRYP_CR_START_MASK);
 249
 250                                 cryp_wait_until_done(device_data);
 251                         }
 252                 }
 253         }
 254
 255         return IRQ_HANDLED;
 256 }
 257
 258 static int mode_is_aes(enum cryp_algo_mode mode)
 259 {
 260         return  CRYP_ALGO_AES_ECB == mode ||
 261                 CRYP_ALGO_AES_CBC == mode ||
 262                 CRYP_ALGO_AES_CTR == mode ||
 263                 CRYP_ALGO_AES_XTS == mode;
 264 }
 265
 266 static int cfg_iv(struct cryp_device_data *device_data, u32 left, u32 right,
 267                   enum cryp_init_vector_index index)
 268 {
 269         struct cryp_init_vector_value vector_value;
 270
 271         dev_dbg(device_data->dev, "[%s]", __func__);
 272
 273         vector_value.init_value_left = left;
 274         vector_value.init_value_right = right;
 275
 276         return cryp_configure_init_vector(device_data,
 277                                           index,
 278                                           vector_value);
 279 }
 280
 281 static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx)
 282 {
 283         int i;
 284         int status = 0;
 285         int num_of_regs = ctx->blocksize / 8;
 286         u32 iv[AES_BLOCK_SIZE / 4];
 287
 288         dev_dbg(device_data->dev, "[%s]", __func__);
 289
 290         /*
 291          * Since we loop on num_of_regs we need to have a check in case
 292          * someone provides an incorrect blocksize which would force calling
 293          * cfg_iv with i greater than 2 which is an error.
 294          */
 295         if (num_of_regs > 2) {
 296                 dev_err(device_data->dev, "[%s] Incorrect blocksize %d",
 297                         __func__, ctx->blocksize);
 298                 return -EINVAL;
 299         }
 300
 301         for (i = 0; i < ctx->blocksize / 4; i++)
 302                 iv[i] = uint8p_to_uint32_be(ctx->iv + i*4);
 303
 304         for (i = 0; i < num_of_regs; i++) {
 305                 status = cfg_iv(device_data, iv[i*2], iv[i*2+1],
 306                                 (enum cryp_init_vector_index) i);
 307                 if (status != 0)
 308                         return status;
 309         }
 310         return status;
 311 }
 312
 313 static int set_key(struct cryp_device_data *device_data,
 314                    u32 left_key,
 315                    u32 right_key,
 316                    enum cryp_key_reg_index index)
 317 {
 318         struct cryp_key_value key_value;
 319         int cryp_error;
 320
 321         dev_dbg(device_data->dev, "[%s]", __func__);
 322
 323         key_value.key_value_left = left_key;
 324         key_value.key_value_right = right_key;
 325
 326         cryp_error = cryp_configure_key_values(device_data,
 327                                                index,
 328                                                key_value);
 329         if (cryp_error != 0)
 330                 dev_err(device_data->dev, "[%s]: "
 331                         "cryp_configure_key_values() failed!", __func__);
 332
 333         return cryp_error;
 334 }
 335
 336 static int cfg_keys(struct cryp_ctx *ctx)
 337 {
 338         int i;
 339         int num_of_regs = ctx->keylen / 8;
 340         u32 swapped_key[CRYP_MAX_KEY_SIZE / 4];
 341         int cryp_error = 0;
 342
 343         dev_dbg(ctx->device->dev, "[%s]", __func__);
 344
 345         if (mode_is_aes(ctx->config.algomode)) {
 346                 swap_words_in_key_and_bits_in_byte((u8 *)ctx->key,
 347                                                    (u8 *)swapped_key,
 348                                                    ctx->keylen);
 349         } else {
 350                 for (i = 0; i < ctx->keylen / 4; i++)
 351                         swapped_key[i] = uint8p_to_uint32_be(ctx->key + i*4);
 352         }
 353
 354         for (i = 0; i < num_of_regs; i++) {
 355                 cryp_error = set_key(ctx->device,
 356                                      *(((u32 *)swapped_key)+i*2),
 357                                      *(((u32 *)swapped_key)+i*2+1),
 358                                      (enum cryp_key_reg_index) i);
 359
 360                 if (cryp_error != 0) {
 361                         dev_err(ctx->device->dev, "[%s]: set_key() failed!",
 362                                         __func__);
 363                         return cryp_error;
 364                 }
 365         }
 366         return cryp_error;
 367 }
 368
 369 static int cryp_setup_context(struct cryp_ctx *ctx,
 370                               struct cryp_device_data *device_data)
 371 {
 372         u32 control_register = CRYP_CR_DEFAULT;
 373
 374         switch (cryp_mode) {
 375         case CRYP_MODE_INTERRUPT:
 376                 writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc);
 377                 break;
 378
 379         case CRYP_MODE_DMA:
 380                 writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr);
 381                 break;
 382
 383         default:
 384                 break;
 385         }
 386
 387         if (ctx->updated == 0) {
 388                 cryp_flush_inoutfifo(device_data);
 389                 if (cfg_keys(ctx) != 0) {
 390                         dev_err(ctx->device->dev, "[%s]: cfg_keys failed!",
 391                                 __func__);
 392                         return -EINVAL;
 393                 }
 394
 395                 if (ctx->iv &&
 396                     CRYP_ALGO_AES_ECB != ctx->config.algomode &&
 397                     CRYP_ALGO_DES_ECB != ctx->config.algomode &&
 398                     CRYP_ALGO_TDES_ECB != ctx->config.algomode) {
 399                         if (cfg_ivs(device_data, ctx) != 0)
 400                                 return -EPERM;
 401                 }
 402
 403                 cryp_set_configuration(device_data, &ctx->config,
 404                                        &control_register);
 405                 add_session_id(ctx);
 406         } else if (ctx->updated == 1 &&
 407                    ctx->session_id != atomic_read(&session_id)) {
 408                 cryp_flush_inoutfifo(device_data);
 409                 cryp_restore_device_context(device_data, &ctx->dev_ctx);
 410
 411                 add_session_id(ctx);
 412                 control_register = ctx->dev_ctx.cr;
 413         } else
 414                 control_register = ctx->dev_ctx.cr;
 415
 416         writel(control_register |
 417                (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS),
 418                &device_data->base->cr);
 419
 420         return 0;
 421 }
 422
 423 static int cryp_get_device_data(struct cryp_ctx *ctx,
 424                                 struct cryp_device_data **device_data)
 425 {
 426         int ret;
 427         struct klist_iter device_iterator;
 428         struct klist_node *device_node;
 429         struct cryp_device_data *local_device_data = NULL;
 430         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 431
 432         /* Wait until a device is available */
 433         ret = down_interruptible(&driver_data.device_allocation);
 434         if (ret)
 435                 return ret;  /* Interrupted */
 436
 437         /* Select a device */
 438         klist_iter_init(&driver_data.device_list, &device_iterator);
 439
 440         device_node = klist_next(&device_iterator);
 441         while (device_node) {
 442                 local_device_data = container_of(device_node,
 443                                            struct cryp_device_data, list_node);
 444                 spin_lock(&local_device_data->ctx_lock);
 445                 /* current_ctx allocates a device, NULL = unallocated */
 446                 if (local_device_data->current_ctx) {
 447                         device_node = klist_next(&device_iterator);
 448                 } else {
 449                         local_device_data->current_ctx = ctx;
 450                         ctx->device = local_device_data;
 451                         spin_unlock(&local_device_data->ctx_lock);
 452                         break;
 453                 }
 454                 spin_unlock(&local_device_data->ctx_lock);
 455         }
 456         klist_iter_exit(&device_iterator);
 457
 458         if (!device_node) {
 459                 /**
 460                  * No free device found.
 461                  * Since we allocated a device with down_interruptible, this
 462                  * should not be able to happen.
 463                  * Number of available devices, which are contained in
 464                  * device_allocation, is therefore decremented by not doing
 465                  * an up(device_allocation).
 466                  */
 467                 return -EBUSY;
 468         }
 469
 470         *device_data = local_device_data;
 471
 472         return 0;
 473 }
 474
 475 static void cryp_dma_setup_channel(struct cryp_device_data *device_data,
 476                                    struct device *dev)
 477 {
 478         dma_cap_zero(device_data->dma.mask);
 479         dma_cap_set(DMA_SLAVE, device_data->dma.mask);
 480
 481         device_data->dma.cfg_mem2cryp = mem_to_engine;
 482         device_data->dma.chan_mem2cryp =
 483                 dma_request_channel(device_data->dma.mask,
 484                                     stedma40_filter,
 485                                     device_data->dma.cfg_mem2cryp);
 486
 487         device_data->dma.cfg_cryp2mem = engine_to_mem;
 488         device_data->dma.chan_cryp2mem =
 489                 dma_request_channel(device_data->dma.mask,
 490                                     stedma40_filter,
 491                                     device_data->dma.cfg_cryp2mem);
 492
 493         init_completion(&device_data->dma.cryp_dma_complete);
 494 }
 495
 496 static void cryp_dma_out_callback(void *data)
 497 {
 498         struct cryp_ctx *ctx = (struct cryp_ctx *) data;
 499         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 500
 501         complete(&ctx->device->dma.cryp_dma_complete);
 502 }
 503
 504 static int cryp_set_dma_transfer(struct cryp_ctx *ctx,
 505                                  struct scatterlist *sg,
 506                                  int len,
 507                                  enum dma_data_direction direction)
 508 {
 509         struct dma_async_tx_descriptor *desc;
 510         struct dma_chan *channel = NULL;
 511         dma_cookie_t cookie;
 512
 513         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 514
 515         if (unlikely(!IS_ALIGNED((u32)sg, 4))) {
 516                 dev_err(ctx->device->dev, "[%s]: Data in sg list isn't "
 517                         "aligned! Addr: 0x%08x", __func__, (u32)sg);
 518                 return -EFAULT;
 519         }
 520
 521         switch (direction) {
 522         case DMA_TO_DEVICE:
 523                 channel = ctx->device->dma.chan_mem2cryp;
 524                 ctx->device->dma.sg_src = sg;
 525                 ctx->device->dma.sg_src_len = dma_map_sg(channel->device->dev,
 526                                                  ctx->device->dma.sg_src,
 527                                                  ctx->device->dma.nents_src,
 528                                                  direction);
 529
 530                 if (!ctx->device->dma.sg_src_len) {
 531                         dev_dbg(ctx->device->dev,
 532                                 "[%s]: Could not map the sg list (TO_DEVICE)",
 533                                 __func__);
 534                         return -EFAULT;
 535                 }
 536
 537                 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
 538                         "(TO_DEVICE)", __func__);
 539
 540                 desc = channel->device->device_prep_slave_sg(channel,
 541                                              ctx->device->dma.sg_src,
 542                                              ctx->device->dma.sg_src_len,
 543                                              direction, DMA_CTRL_ACK, NULL);
 544                 break;
 545
 546         case DMA_FROM_DEVICE:
 547                 channel = ctx->device->dma.chan_cryp2mem;
 548                 ctx->device->dma.sg_dst = sg;
 549                 ctx->device->dma.sg_dst_len = dma_map_sg(channel->device->dev,
 550                                                  ctx->device->dma.sg_dst,
 551                                                  ctx->device->dma.nents_dst,
 552                                                  direction);
 553
 554                 if (!ctx->device->dma.sg_dst_len) {
 555                         dev_dbg(ctx->device->dev,
 556                                 "[%s]: Could not map the sg list (FROM_DEVICE)",
 557                                 __func__);
 558                         return -EFAULT;
 559                 }
 560
 561                 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
 562                         "(FROM_DEVICE)", __func__);
 563
 564                 desc = channel->device->device_prep_slave_sg(channel,
 565                                              ctx->device->dma.sg_dst,
 566                                              ctx->device->dma.sg_dst_len,
 567                                              direction,
 568                                              DMA_CTRL_ACK |
 569                                              DMA_PREP_INTERRUPT, NULL);
 570
 571                 desc->callback = cryp_dma_out_callback;
 572                 desc->callback_param = ctx;
 573                 break;
 574
 575         default:
 576                 dev_dbg(ctx->device->dev, "[%s]: Invalid DMA direction",
 577                         __func__);
 578                 return -EFAULT;
 579         }
 580
 581         cookie = desc->tx_submit(desc);
 582         dma_async_issue_pending(channel);
 583
 584         return 0;
 585 }
 586
 587 static void cryp_dma_done(struct cryp_ctx *ctx)
 588 {
 589         struct dma_chan *chan;
 590
 591         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 592
 593         chan = ctx->device->dma.chan_mem2cryp;
 594         chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
 595         dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_src,
 596                      ctx->device->dma.sg_src_len, DMA_TO_DEVICE);
 597
 598         chan = ctx->device->dma.chan_cryp2mem;
 599         chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
 600         dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_dst,
 601                      ctx->device->dma.sg_dst_len, DMA_FROM_DEVICE);
 602 }
 603
 604 static int cryp_dma_write(struct cryp_ctx *ctx, struct scatterlist *sg,
 605                           int len)
 606 {
 607         int error = cryp_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
 608         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 609
 610         if (error) {
 611                 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
 612                         "failed", __func__);
 613                 return error;
 614         }
 615
 616         return len;
 617 }
 618
 619 static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len)
 620 {
 621         int error = cryp_set_dma_transfer(ctx, sg, len, DMA_FROM_DEVICE);
 622         if (error) {
 623                 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
 624                         "failed", __func__);
 625                 return error;
 626         }
 627
 628         return len;
 629 }
 630
 631 static void cryp_polling_mode(struct cryp_ctx *ctx,
 632                               struct cryp_device_data *device_data)
 633 {
 634         int len = ctx->blocksize / BYTES_PER_WORD;
 635         int remaining_length = ctx->datalen;
 636         u32 *indata = (u32 *)ctx->indata;
 637         u32 *outdata = (u32 *)ctx->outdata;
 638
 639         while (remaining_length > 0) {
 640                 writesl(&device_data->base->din, indata, len);
 641                 indata += len;
 642                 remaining_length -= (len * BYTES_PER_WORD);
 643                 cryp_wait_until_done(device_data);
 644
 645                 readsl(&device_data->base->dout, outdata, len);
 646                 outdata += len;
 647                 cryp_wait_until_done(device_data);
 648         }
 649 }
 650
 651 static int cryp_disable_power(struct device *dev,
 652                               struct cryp_device_data *device_data,
 653                               bool save_device_context)
 654 {
 655         int ret = 0;
 656
 657         dev_dbg(dev, "[%s]", __func__);
 658
 659         spin_lock(&device_data->power_state_spinlock);
 660         if (!device_data->power_state)
 661                 goto out;
 662
 663         spin_lock(&device_data->ctx_lock);
 664         if (save_device_context && device_data->current_ctx) {
 665                 cryp_save_device_context(device_data,
 666                                 &device_data->current_ctx->dev_ctx,
 667                                 cryp_mode);
 668                 device_data->restore_dev_ctx = true;
 669         }
 670         spin_unlock(&device_data->ctx_lock);
 671
 672         clk_disable(device_data->clk);
 673         ret = regulator_disable(device_data->pwr_regulator);
 674         if (ret)
 675                 dev_err(dev, "[%s]: "
 676                                 "regulator_disable() failed!",
 677                                 __func__);
 678
 679         device_data->power_state = false;
 680
 681 out:
 682         spin_unlock(&device_data->power_state_spinlock);
 683
 684         return ret;
 685 }
 686
 687 static int cryp_enable_power(
 688                 struct device *dev,
 689                 struct cryp_device_data *device_data,
 690                 bool restore_device_context)
 691 {
 692         int ret = 0;
 693
 694         dev_dbg(dev, "[%s]", __func__);
 695
 696         spin_lock(&device_data->power_state_spinlock);
 697         if (!device_data->power_state) {
 698                 ret = regulator_enable(device_data->pwr_regulator);
 699                 if (ret) {
 700                         dev_err(dev, "[%s]: regulator_enable() failed!",
 701                                         __func__);
 702                         goto out;
 703                 }
 704
 705                 ret = clk_enable(device_data->clk);
 706                 if (ret) {
 707                         dev_err(dev, "[%s]: clk_enable() failed!",
 708                                         __func__);
 709                         regulator_disable(device_data->pwr_regulator);
 710                         goto out;
 711                 }
 712                 device_data->power_state = true;
 713         }
 714
 715         if (device_data->restore_dev_ctx) {
 716                 spin_lock(&device_data->ctx_lock);
 717                 if (restore_device_context && device_data->current_ctx) {
 718                         device_data->restore_dev_ctx = false;
 719                         cryp_restore_device_context(device_data,
 720                                         &device_data->current_ctx->dev_ctx);
 721                 }
 722                 spin_unlock(&device_data->ctx_lock);
 723         }
 724 out:
 725         spin_unlock(&device_data->power_state_spinlock);
 726
 727         return ret;
 728 }
 729
 730 static int hw_crypt_noxts(struct cryp_ctx *ctx,
 731                           struct cryp_device_data *device_data)
 732 {
 733         int ret = 0;
 734
 735         const u8 *indata = ctx->indata;
 736         u8 *outdata = ctx->outdata;
 737         u32 datalen = ctx->datalen;
 738         u32 outlen = datalen;
 739
 740         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 741
 742         ctx->outlen = ctx->datalen;
 743
 744         if (unlikely(!IS_ALIGNED((u32)indata, 4))) {
 745                 pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: "
 746                          "0x%08x", __func__, (u32)indata);
 747                 return -EINVAL;
 748         }
 749
 750         ret = cryp_setup_context(ctx, device_data);
 751
 752         if (ret)
 753                 goto out;
 754
 755         if (cryp_mode == CRYP_MODE_INTERRUPT) {
 756                 cryp_enable_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO |
 757                                     CRYP_IRQ_SRC_OUTPUT_FIFO);
 758
 759                 /*
 760                  * ctx->outlen is decremented in the cryp_interrupt_handler
 761                  * function. We had to add cpu_relax() (barrier) to make sure
 762                  * that gcc didn't optimze away this variable.
 763                  */
 764                 while (ctx->outlen > 0)
 765                         cpu_relax();
 766         } else if (cryp_mode == CRYP_MODE_POLLING ||
 767                    cryp_mode == CRYP_MODE_DMA) {
 768                 /*
 769                  * The reason for having DMA in this if case is that if we are
 770                  * running cryp_mode = 2, then we separate DMA routines for
 771                  * handling cipher/plaintext > blocksize, except when
 772                  * running the normal CRYPTO_ALG_TYPE_CIPHER, then we still use
 773                  * the polling mode. Overhead of doing DMA setup eats up the
 774                  * benefits using it.
 775                  */
 776                 cryp_polling_mode(ctx, device_data);
 777         } else {
 778                 dev_err(ctx->device->dev, "[%s]: Invalid operation mode!",
 779                         __func__);
 780                 ret = -EPERM;
 781                 goto out;
 782         }
 783
 784         cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
 785         ctx->updated = 1;
 786
 787 out:
 788         ctx->indata = indata;
 789         ctx->outdata = outdata;
 790         ctx->datalen = datalen;
 791         ctx->outlen = outlen;
 792
 793         return ret;
 794 }
 795
 796 static int get_nents(struct scatterlist *sg, int nbytes)
 797 {
 798         int nents = 0;
 799
 800         while (nbytes > 0) {
 801                 nbytes -= sg->length;
 802                 sg = scatterwalk_sg_next(sg);
 803                 nents++;
 804         }
 805
 806         return nents;
 807 }
 808
 809 static int ablk_dma_crypt(struct ablkcipher_request *areq)
 810 {
 811         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
 812         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 813         struct cryp_device_data *device_data;
 814
 815         int bytes_written = 0;
 816         int bytes_read = 0;
 817         int ret;
 818
 819         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 820
 821         ctx->datalen = areq->nbytes;
 822         ctx->outlen = areq->nbytes;
 823
 824         ret = cryp_get_device_data(ctx, &device_data);
 825         if (ret)
 826                 return ret;
 827
 828         ret = cryp_setup_context(ctx, device_data);
 829         if (ret)
 830                 goto out;
 831
 832         /* We have the device now, so store the nents in the dma struct. */
 833         ctx->device->dma.nents_src = get_nents(areq->src, ctx->datalen);
 834         ctx->device->dma.nents_dst = get_nents(areq->dst, ctx->outlen);
 835
 836         /* Enable DMA in- and output. */
 837         cryp_configure_for_dma(device_data, CRYP_DMA_ENABLE_BOTH_DIRECTIONS);
 838
 839         bytes_written = cryp_dma_write(ctx, areq->src, ctx->datalen);
 840         bytes_read = cryp_dma_read(ctx, areq->dst, bytes_written);
 841
 842         wait_for_completion(&ctx->device->dma.cryp_dma_complete);
 843         cryp_dma_done(ctx);
 844
 845         cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
 846         ctx->updated = 1;
 847
 848 out:
 849         spin_lock(&device_data->ctx_lock);
 850         device_data->current_ctx = NULL;
 851         ctx->device = NULL;
 852         spin_unlock(&device_data->ctx_lock);
 853
 854         /*
 855          * The down_interruptible part for this semaphore is called in
 856          * cryp_get_device_data.
 857          */
 858         up(&driver_data.device_allocation);
 859
 860         if (unlikely(bytes_written != bytes_read))
 861                 return -EPERM;
 862
 863         return 0;
 864 }
 865
 866 static int ablk_crypt(struct ablkcipher_request *areq)
 867 {
 868         struct ablkcipher_walk walk;
 869         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
 870         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 871         struct cryp_device_data *device_data;
 872         unsigned long src_paddr;
 873         unsigned long dst_paddr;
 874         int ret;
 875         int nbytes;
 876
 877         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 878
 879         ret = cryp_get_device_data(ctx, &device_data);
 880         if (ret)
 881                 goto out;
 882
 883         ablkcipher_walk_init(&walk, areq->dst, areq->src, areq->nbytes);
 884         ret = ablkcipher_walk_phys(areq, &walk);
 885
 886         if (ret) {
 887                 pr_err(DEV_DBG_NAME "[%s]: ablkcipher_walk_phys() failed!",
 888                         __func__);
 889                 goto out;
 890         }
 891
 892         while ((nbytes = walk.nbytes) > 0) {
 893                 ctx->iv = walk.iv;
 894                 src_paddr = (page_to_phys(walk.src.page) + walk.src.offset);
 895                 ctx->indata = phys_to_virt(src_paddr);
 896
 897                 dst_paddr = (page_to_phys(walk.dst.page) + walk.dst.offset);
 898                 ctx->outdata = phys_to_virt(dst_paddr);
 899
 900                 ctx->datalen = nbytes - (nbytes % ctx->blocksize);
 901
 902                 ret = hw_crypt_noxts(ctx, device_data);
 903                 if (ret)
 904                         goto out;
 905
 906                 nbytes -= ctx->datalen;
 907                 ret = ablkcipher_walk_done(areq, &walk, nbytes);
 908                 if (ret)
 909                         goto out;
 910         }
 911         ablkcipher_walk_complete(&walk);
 912
 913 out:
 914         /* Release the device */
 915         spin_lock(&device_data->ctx_lock);
 916         device_data->current_ctx = NULL;
 917         ctx->device = NULL;
 918         spin_unlock(&device_data->ctx_lock);
 919
 920         /*
 921          * The down_interruptible part for this semaphore is called in
 922          * cryp_get_device_data.
 923          */
 924         up(&driver_data.device_allocation);
 925
 926         return ret;
 927 }
 928
 929 static int aes_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
 930                                  const u8 *key, unsigned int keylen)
 931 {
 932         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 933         u32 *flags = &cipher->base.crt_flags;
 934
 935         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 936
 937         switch (keylen) {
 938         case AES_KEYSIZE_128:
 939                 ctx->config.keysize = CRYP_KEY_SIZE_128;
 940                 break;
 941
 942         case AES_KEYSIZE_192:
 943                 ctx->config.keysize = CRYP_KEY_SIZE_192;
 944                 break;
 945
 946         case AES_KEYSIZE_256:
 947                 ctx->config.keysize = CRYP_KEY_SIZE_256;
 948                 break;
 949
 950         default:
 951                 pr_err(DEV_DBG_NAME "[%s]: Unknown keylen!", __func__);
 952                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 953                 return -EINVAL;
 954         }
 955
 956         memcpy(ctx->key, key, keylen);
 957         ctx->keylen = keylen;
 958
 959         ctx->updated = 0;
 960
 961         return 0;
 962 }
 963
 964 static int des_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
 965                                  const u8 *key, unsigned int keylen)
 966 {
 967         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 968         u32 *flags = &cipher->base.crt_flags;
 969         u32 tmp[DES_EXPKEY_WORDS];
 970         int ret;
 971
 972         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 973         if (keylen != DES_KEY_SIZE) {
 974                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 975                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN",
 976                                 __func__);
 977                 return -EINVAL;
 978         }
 979
 980         ret = des_ekey(tmp, key);
 981         if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 982                 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
 983                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY",
 984                                 __func__);
 985                 return -EINVAL;
 986         }
 987
 988         memcpy(ctx->key, key, keylen);
 989         ctx->keylen = keylen;
 990
 991         ctx->updated = 0;
 992         return 0;
 993 }
 994
 995 static int des3_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
 996                                   const u8 *key, unsigned int keylen)
 997 {
 998         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 999         u32 *flags = &cipher->base.crt_flags;
1000         const u32 *K = (const u32 *)key;
1001         u32 tmp[DES3_EDE_EXPKEY_WORDS];
1002         int i, ret;
1003
1004         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1005         if (keylen != DES3_EDE_KEY_SIZE) {
1006                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
1007                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN",
1008                                 __func__);
1009                 return -EINVAL;
1010         }
1011
1012         /* Checking key interdependency for weak key detection. */
1013         if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
1014                                 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
1015                         (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
1016                 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
1017                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY",
1018                                 __func__);
1019                 return -EINVAL;
1020         }
1021         for (i = 0; i < 3; i++) {
1022                 ret = des_ekey(tmp, key + i*DES_KEY_SIZE);
1023                 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
1024                         *flags |= CRYPTO_TFM_RES_WEAK_KEY;
1025                         pr_debug(DEV_DBG_NAME " [%s]: "
1026                                         "CRYPTO_TFM_REQ_WEAK_KEY", __func__);
1027                         return -EINVAL;
1028                 }
1029         }
1030
1031         memcpy(ctx->key, key, keylen);
1032         ctx->keylen = keylen;
1033
1034         ctx->updated = 0;
1035         return 0;
1036 }
1037
1038 static int cryp_blk_encrypt(struct ablkcipher_request *areq)
1039 {
1040         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1041         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1042
1043         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1044
1045         ctx->config.algodir = CRYP_ALGORITHM_ENCRYPT;
1046
1047         /*
1048          * DMA does not work for DES due to a hw bug */
1049         if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1050                 return ablk_dma_crypt(areq);
1051
1052         /* For everything except DMA, we run the non DMA version. */
1053         return ablk_crypt(areq);
1054 }
1055
1056 static int cryp_blk_decrypt(struct ablkcipher_request *areq)
1057 {
1058         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1059         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1060
1061         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1062
1063         ctx->config.algodir = CRYP_ALGORITHM_DECRYPT;
1064
1065         /* DMA does not work for DES due to a hw bug */
1066         if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1067                 return ablk_dma_crypt(areq);
1068
1069         /* For everything except DMA, we run the non DMA version. */
1070         return ablk_crypt(areq);
1071 }
1072
1073 struct cryp_algo_template {
1074         enum cryp_algo_mode algomode;
1075         struct crypto_alg crypto;
1076 };
1077
1078 static int cryp_cra_init(struct crypto_tfm *tfm)
1079 {
1080         struct cryp_ctx *ctx = crypto_tfm_ctx(tfm);
1081         struct crypto_alg *alg = tfm->__crt_alg;
1082         struct cryp_algo_template *cryp_alg = container_of(alg,
1083                         struct cryp_algo_template,
1084                         crypto);
1085
1086         ctx->config.algomode = cryp_alg->algomode;
1087         ctx->blocksize = crypto_tfm_alg_blocksize(tfm);
1088
1089         return 0;
1090 }
1091
1092 static struct cryp_algo_template cryp_algs[] = {
1093         {
1094                 .algomode = CRYP_ALGO_AES_ECB,
1095                 .crypto = {
1096                         .cra_name = "aes",
1097                         .cra_driver_name = "aes-ux500",
1098                         .cra_priority = 300,
1099                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1100                                         CRYPTO_ALG_ASYNC,
1101                         .cra_blocksize = AES_BLOCK_SIZE,
1102                         .cra_ctxsize = sizeof(struct cryp_ctx),
1103                         .cra_alignmask = 3,
1104                         .cra_type = &crypto_ablkcipher_type,
1105                         .cra_init = cryp_cra_init,
1106                         .cra_module = THIS_MODULE,
1107                         .cra_u = {
1108                                 .ablkcipher = {
1109                                         .min_keysize = AES_MIN_KEY_SIZE,
1110                                         .max_keysize = AES_MAX_KEY_SIZE,
1111                                         .setkey = aes_ablkcipher_setkey,
1112                                         .encrypt = cryp_blk_encrypt,
1113                                         .decrypt = cryp_blk_decrypt
1114                                 }
1115                         }
1116                 }
1117         },
1118         {
1119                 .algomode = CRYP_ALGO_AES_ECB,
1120                 .crypto = {
1121                         .cra_name = "ecb(aes)",
1122                         .cra_driver_name = "ecb-aes-ux500",
1123                         .cra_priority = 300,
1124                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1125                                         CRYPTO_ALG_ASYNC,
1126                         .cra_blocksize = AES_BLOCK_SIZE,
1127                         .cra_ctxsize = sizeof(struct cryp_ctx),
1128                         .cra_alignmask = 3,
1129                         .cra_type = &crypto_ablkcipher_type,
1130                         .cra_init = cryp_cra_init,
1131                         .cra_module = THIS_MODULE,
1132                         .cra_u = {
1133                                 .ablkcipher = {
1134                                         .min_keysize = AES_MIN_KEY_SIZE,
1135                                         .max_keysize = AES_MAX_KEY_SIZE,
1136                                         .setkey = aes_ablkcipher_setkey,
1137                                         .encrypt = cryp_blk_encrypt,
1138                                         .decrypt = cryp_blk_decrypt,
1139                                 }
1140                         }
1141                 }
1142         },
1143         {
1144                 .algomode = CRYP_ALGO_AES_CBC,
1145                 .crypto = {
1146                         .cra_name = "cbc(aes)",
1147                         .cra_driver_name = "cbc-aes-ux500",
1148                         .cra_priority = 300,
1149                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1150                                         CRYPTO_ALG_ASYNC,
1151                         .cra_blocksize = AES_BLOCK_SIZE,
1152                         .cra_ctxsize = sizeof(struct cryp_ctx),
1153                         .cra_alignmask = 3,
1154                         .cra_type = &crypto_ablkcipher_type,
1155                         .cra_init = cryp_cra_init,
1156                         .cra_module = THIS_MODULE,
1157                         .cra_u = {
1158                                 .ablkcipher = {
1159                                         .min_keysize = AES_MIN_KEY_SIZE,
1160                                         .max_keysize = AES_MAX_KEY_SIZE,
1161                                         .setkey = aes_ablkcipher_setkey,
1162                                         .encrypt = cryp_blk_encrypt,
1163                                         .decrypt = cryp_blk_decrypt,
1164                                         .ivsize = AES_BLOCK_SIZE,
1165                                 }
1166                         }
1167                 }
1168         },
1169         {
1170                 .algomode = CRYP_ALGO_AES_CTR,
1171                 .crypto = {
1172                         .cra_name = "ctr(aes)",
1173                         .cra_driver_name = "ctr-aes-ux500",
1174                         .cra_priority = 300,
1175                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1176                                                 CRYPTO_ALG_ASYNC,
1177                         .cra_blocksize = AES_BLOCK_SIZE,
1178                         .cra_ctxsize = sizeof(struct cryp_ctx),
1179                         .cra_alignmask = 3,
1180                         .cra_type = &crypto_ablkcipher_type,
1181                         .cra_init = cryp_cra_init,
1182                         .cra_module = THIS_MODULE,
1183                         .cra_u = {
1184                                 .ablkcipher = {
1185                                         .min_keysize = AES_MIN_KEY_SIZE,
1186                                         .max_keysize = AES_MAX_KEY_SIZE,
1187                                         .setkey = aes_ablkcipher_setkey,
1188                                         .encrypt = cryp_blk_encrypt,
1189                                         .decrypt = cryp_blk_decrypt,
1190                                         .ivsize = AES_BLOCK_SIZE,
1191                                 }
1192                         }
1193                 }
1194         },
1195         {
1196                 .algomode = CRYP_ALGO_DES_ECB,
1197                 .crypto = {
1198                         .cra_name = "des",
1199                         .cra_driver_name = "des-ux500",
1200                         .cra_priority = 300,
1201                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1202                                                 CRYPTO_ALG_ASYNC,
1203                         .cra_blocksize = DES_BLOCK_SIZE,
1204                         .cra_ctxsize = sizeof(struct cryp_ctx),
1205                         .cra_alignmask = 3,
1206                         .cra_type = &crypto_ablkcipher_type,
1207                         .cra_init = cryp_cra_init,
1208                         .cra_module = THIS_MODULE,
1209                         .cra_u = {
1210                                 .ablkcipher = {
1211                                         .min_keysize = DES_KEY_SIZE,
1212                                         .max_keysize = DES_KEY_SIZE,
1213                                         .setkey = des_ablkcipher_setkey,
1214                                         .encrypt = cryp_blk_encrypt,
1215                                         .decrypt = cryp_blk_decrypt
1216                                 }
1217                         }
1218                 }
1219
1220         },
1221         {
1222                 .algomode = CRYP_ALGO_TDES_ECB,
1223                 .crypto = {
1224                         .cra_name = "des3_ede",
1225                         .cra_driver_name = "des3_ede-ux500",
1226                         .cra_priority = 300,
1227                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1228                                                 CRYPTO_ALG_ASYNC,
1229                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1230                         .cra_ctxsize = sizeof(struct cryp_ctx),
1231                         .cra_alignmask = 3,
1232                         .cra_type = &crypto_ablkcipher_type,
1233                         .cra_init = cryp_cra_init,
1234                         .cra_module = THIS_MODULE,
1235                         .cra_u = {
1236                                 .ablkcipher = {
1237                                         .min_keysize = DES3_EDE_KEY_SIZE,
1238                                         .max_keysize = DES3_EDE_KEY_SIZE,
1239                                         .setkey = des_ablkcipher_setkey,
1240                                         .encrypt = cryp_blk_encrypt,
1241                                         .decrypt = cryp_blk_decrypt
1242                                 }
1243                         }
1244                 }
1245         },
1246         {
1247                 .algomode = CRYP_ALGO_DES_ECB,
1248                 .crypto = {
1249                         .cra_name = "ecb(des)",
1250                         .cra_driver_name = "ecb-des-ux500",
1251                         .cra_priority = 300,
1252                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1253                                         CRYPTO_ALG_ASYNC,
1254                         .cra_blocksize = DES_BLOCK_SIZE,
1255                         .cra_ctxsize = sizeof(struct cryp_ctx),
1256                         .cra_alignmask = 3,
1257                         .cra_type = &crypto_ablkcipher_type,
1258                         .cra_init = cryp_cra_init,
1259                         .cra_module = THIS_MODULE,
1260                         .cra_u = {
1261                                 .ablkcipher = {
1262                                         .min_keysize = DES_KEY_SIZE,
1263                                         .max_keysize = DES_KEY_SIZE,
1264                                         .setkey = des_ablkcipher_setkey,
1265                                         .encrypt = cryp_blk_encrypt,
1266                                         .decrypt = cryp_blk_decrypt,
1267                                 }
1268                         }
1269                 }
1270         },
1271         {
1272                 .algomode = CRYP_ALGO_TDES_ECB,
1273                 .crypto = {
1274                         .cra_name = "ecb(des3_ede)",
1275                         .cra_driver_name = "ecb-des3_ede-ux500",
1276                         .cra_priority = 300,
1277                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1278                                         CRYPTO_ALG_ASYNC,
1279                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1280                         .cra_ctxsize = sizeof(struct cryp_ctx),
1281                         .cra_alignmask = 3,
1282                         .cra_type = &crypto_ablkcipher_type,
1283                         .cra_init = cryp_cra_init,
1284                         .cra_module = THIS_MODULE,
1285                         .cra_u = {
1286                                 .ablkcipher = {
1287                                         .min_keysize = DES3_EDE_KEY_SIZE,
1288                                         .max_keysize = DES3_EDE_KEY_SIZE,
1289                                         .setkey = des3_ablkcipher_setkey,
1290                                         .encrypt = cryp_blk_encrypt,
1291                                         .decrypt = cryp_blk_decrypt,
1292                                 }
1293                         }
1294                 }
1295         },
1296         {
1297                 .algomode = CRYP_ALGO_DES_CBC,
1298                 .crypto = {
1299                         .cra_name = "cbc(des)",
1300                         .cra_driver_name = "cbc-des-ux500",
1301                         .cra_priority = 300,
1302                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1303                                         CRYPTO_ALG_ASYNC,
1304                         .cra_blocksize = DES_BLOCK_SIZE,
1305                         .cra_ctxsize = sizeof(struct cryp_ctx),
1306                         .cra_alignmask = 3,
1307                         .cra_type = &crypto_ablkcipher_type,
1308                         .cra_init = cryp_cra_init,
1309                         .cra_module = THIS_MODULE,
1310                         .cra_u = {
1311                                 .ablkcipher = {
1312                                         .min_keysize = DES_KEY_SIZE,
1313                                         .max_keysize = DES_KEY_SIZE,
1314                                         .setkey = des_ablkcipher_setkey,
1315                                         .encrypt = cryp_blk_encrypt,
1316                                         .decrypt = cryp_blk_decrypt,
1317                                 }
1318                         }
1319                 }
1320         },
1321         {
1322                 .algomode = CRYP_ALGO_TDES_CBC,
1323                 .crypto = {
1324                         .cra_name = "cbc(des3_ede)",
1325                         .cra_driver_name = "cbc-des3_ede-ux500",
1326                         .cra_priority = 300,
1327                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1328                                         CRYPTO_ALG_ASYNC,
1329                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1330                         .cra_ctxsize = sizeof(struct cryp_ctx),
1331                         .cra_alignmask = 3,
1332                         .cra_type = &crypto_ablkcipher_type,
1333                         .cra_init = cryp_cra_init,
1334                         .cra_module = THIS_MODULE,
1335                         .cra_u = {
1336                                 .ablkcipher = {
1337                                         .min_keysize = DES3_EDE_KEY_SIZE,
1338                                         .max_keysize = DES3_EDE_KEY_SIZE,
1339                                         .setkey = des3_ablkcipher_setkey,
1340                                         .encrypt = cryp_blk_encrypt,
1341                                         .decrypt = cryp_blk_decrypt,
1342                                         .ivsize = DES3_EDE_BLOCK_SIZE,
1343                                 }
1344                         }
1345                 }
1346         }
1347 };
1348
1349 /**
1350  * cryp_algs_register_all -
1351  */
1352 static int cryp_algs_register_all(void)
1353 {
1354         int ret;
1355         int i;
1356         int count;
1357
1358         pr_debug("[%s]", __func__);
1359
1360         for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) {
1361                 ret = crypto_register_alg(&cryp_algs[i].crypto);
1362                 if (ret) {
1363                         count = i;
1364                         pr_err("[%s] alg registration failed",
1365                                         cryp_algs[i].crypto.cra_driver_name);
1366                         goto unreg;
1367                 }
1368         }
1369         return 0;
1370 unreg:
1371         for (i = 0; i < count; i++)
1372                 crypto_unregister_alg(&cryp_algs[i].crypto);
1373         return ret;
1374 }
1375
1376 /**
1377  * cryp_algs_unregister_all -
1378  */
1379 static void cryp_algs_unregister_all(void)
1380 {
1381         int i;
1382
1383         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1384
1385         for (i = 0; i < ARRAY_SIZE(cryp_algs); i++)
1386                 crypto_unregister_alg(&cryp_algs[i].crypto);
1387 }
1388
1389 static int ux500_cryp_probe(struct platform_device *pdev)
1390 {
1391         int ret;
1392         int cryp_error = 0;
1393         struct resource *res = NULL;
1394         struct resource *res_irq = NULL;
1395         struct cryp_device_data *device_data;
1396         struct cryp_protection_config prot = {
1397                 .privilege_access = CRYP_STATE_ENABLE
1398         };
1399         struct device *dev = &pdev->dev;
1400
1401         dev_dbg(dev, "[%s]", __func__);
1402         device_data = kzalloc(sizeof(struct cryp_device_data), GFP_ATOMIC);
1403         if (!device_data) {
1404                 dev_err(dev, "[%s]: kzalloc() failed!", __func__);
1405                 ret = -ENOMEM;
1406                 goto out;
1407         }
1408
1409         device_data->dev = dev;
1410         device_data->current_ctx = NULL;
1411
1412         /* Grab the DMA configuration from platform data. */
1413         mem_to_engine = &((struct cryp_platform_data *)
1414                          dev->platform_data)->mem_to_engine;
1415         engine_to_mem = &((struct cryp_platform_data *)
1416                          dev->platform_data)->engine_to_mem;
1417
1418         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1419         if (!res) {
1420                 dev_err(dev, "[%s]: platform_get_resource() failed",
1421                                 __func__);
1422                 ret = -ENODEV;
1423                 goto out_kfree;
1424         }
1425
1426         res = request_mem_region(res->start, resource_size(res), pdev->name);
1427         if (res == NULL) {
1428                 dev_err(dev, "[%s]: request_mem_region() failed",
1429                                 __func__);
1430                 ret = -EBUSY;
1431                 goto out_kfree;
1432         }
1433
1434         device_data->base = ioremap(res->start, resource_size(res));
1435         if (!device_data->base) {
1436                 dev_err(dev, "[%s]: ioremap failed!", __func__);
1437                 ret = -ENOMEM;
1438                 goto out_free_mem;
1439         }
1440
1441         spin_lock_init(&device_data->ctx_lock);
1442         spin_lock_init(&device_data->power_state_spinlock);
1443
1444         /* Enable power for CRYP hardware block */
1445         device_data->pwr_regulator = regulator_get(&pdev->dev, "v-ape");
1446         if (IS_ERR(device_data->pwr_regulator)) {
1447                 dev_err(dev, "[%s]: could not get cryp regulator", __func__);
1448                 ret = PTR_ERR(device_data->pwr_regulator);
1449                 device_data->pwr_regulator = NULL;
1450                 goto out_unmap;
1451         }
1452
1453         /* Enable the clk for CRYP hardware block */
1454         device_data->clk = clk_get(&pdev->dev, NULL);
1455         if (IS_ERR(device_data->clk)) {
1456                 dev_err(dev, "[%s]: clk_get() failed!", __func__);
1457                 ret = PTR_ERR(device_data->clk);
1458                 goto out_regulator;
1459         }
1460
1461         /* Enable device power (and clock) */
1462         ret = cryp_enable_power(device_data->dev, device_data, false);
1463         if (ret) {
1464                 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1465                 goto out_clk;
1466         }
1467
1468         cryp_error = cryp_check(device_data);
1469         if (cryp_error != 0) {
1470                 dev_err(dev, "[%s]: cryp_init() failed!", __func__);
1471                 ret = -EINVAL;
1472                 goto out_power;
1473         }
1474
1475         cryp_error = cryp_configure_protection(device_data, &prot);
1476         if (cryp_error != 0) {
1477                 dev_err(dev, "[%s]: cryp_configure_protection() failed!",
1478                         __func__);
1479                 ret = -EINVAL;
1480                 goto out_power;
1481         }
1482
1483         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1484         if (!res_irq) {
1485                 dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable",
1486                         __func__);
1487                 ret = -ENODEV;
1488                 goto out_power;
1489         }
1490
1491         ret = request_irq(res_irq->start,
1492                           cryp_interrupt_handler,
1493                           0,
1494                           "cryp1",
1495                           device_data);
1496         if (ret) {
1497                 dev_err(dev, "[%s]: Unable to request IRQ", __func__);
1498                 goto out_power;
1499         }
1500
1501         if (cryp_mode == CRYP_MODE_DMA)
1502                 cryp_dma_setup_channel(device_data, dev);
1503
1504         platform_set_drvdata(pdev, device_data);
1505
1506         /* Put the new device into the device list... */
1507         klist_add_tail(&device_data->list_node, &driver_data.device_list);
1508
1509         /* ... and signal that a new device is available. */
1510         up(&driver_data.device_allocation);
1511
1512         atomic_set(&session_id, 1);
1513
1514         ret = cryp_algs_register_all();
1515         if (ret) {
1516                 dev_err(dev, "[%s]: cryp_algs_register_all() failed!",
1517                         __func__);
1518                 goto out_power;
1519         }
1520
1521         return 0;
1522
1523 out_power:
1524         cryp_disable_power(device_data->dev, device_data, false);
1525
1526 out_clk:
1527         clk_put(device_data->clk);
1528
1529 out_regulator:
1530         regulator_put(device_data->pwr_regulator);
1531
1532 out_unmap:
1533         iounmap(device_data->base);
1534
1535 out_free_mem:
1536         release_mem_region(res->start, resource_size(res));
1537
1538 out_kfree:
1539         kfree(device_data);
1540 out:
1541         return ret;
1542 }
1543
1544 static int ux500_cryp_remove(struct platform_device *pdev)
1545 {
1546         struct resource *res = NULL;
1547         struct resource *res_irq = NULL;
1548         struct cryp_device_data *device_data;
1549
1550         dev_dbg(&pdev->dev, "[%s]", __func__);
1551         device_data = platform_get_drvdata(pdev);
1552         if (!device_data) {
1553                 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1554                         __func__);
1555                 return -ENOMEM;
1556         }
1557
1558         /* Try to decrease the number of available devices. */
1559         if (down_trylock(&driver_data.device_allocation))
1560                 return -EBUSY;
1561
1562         /* Check that the device is free */
1563         spin_lock(&device_data->ctx_lock);
1564         /* current_ctx allocates a device, NULL = unallocated */
1565         if (device_data->current_ctx) {
1566                 /* The device is busy */
1567                 spin_unlock(&device_data->ctx_lock);
1568                 /* Return the device to the pool. */
1569                 up(&driver_data.device_allocation);
1570                 return -EBUSY;
1571         }
1572
1573         spin_unlock(&device_data->ctx_lock);
1574
1575         /* Remove the device from the list */
1576         if (klist_node_attached(&device_data->list_node))
1577                 klist_remove(&device_data->list_node);
1578
1579         /* If this was the last device, remove the services */
1580         if (list_empty(&driver_data.device_list.k_list))
1581                 cryp_algs_unregister_all();
1582
1583         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1584         if (!res_irq)
1585                 dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable",
1586                         __func__);
1587         else {
1588                 disable_irq(res_irq->start);
1589                 free_irq(res_irq->start, device_data);
1590         }
1591
1592         if (cryp_disable_power(&pdev->dev, device_data, false))
1593                 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1594                         __func__);
1595
1596         clk_put(device_data->clk);
1597         regulator_put(device_data->pwr_regulator);
1598
1599         iounmap(device_data->base);
1600
1601         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1602         if (res)
1603                 release_mem_region(res->start, resource_size(res));
1604
1605         kfree(device_data);
1606
1607         return 0;
1608 }
1609
1610 static void ux500_cryp_shutdown(struct platform_device *pdev)
1611 {
1612         struct resource *res_irq = NULL;
1613         struct cryp_device_data *device_data;
1614
1615         dev_dbg(&pdev->dev, "[%s]", __func__);
1616
1617         device_data = platform_get_drvdata(pdev);
1618         if (!device_data) {
1619                 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1620                         __func__);
1621                 return;
1622         }
1623
1624         /* Check that the device is free */
1625         spin_lock(&device_data->ctx_lock);
1626         /* current_ctx allocates a device, NULL = unallocated */
1627         if (!device_data->current_ctx) {
1628                 if (down_trylock(&driver_data.device_allocation))
1629                         dev_dbg(&pdev->dev, "[%s]: Cryp still in use!"
1630                                 "Shutting down anyway...", __func__);
1631                 /**
1632                  * (Allocate the device)
1633                  * Need to set this to non-null (dummy) value,
1634                  * to avoid usage if context switching.
1635                  */
1636                 device_data->current_ctx++;
1637         }
1638         spin_unlock(&device_data->ctx_lock);
1639
1640         /* Remove the device from the list */
1641         if (klist_node_attached(&device_data->list_node))
1642                 klist_remove(&device_data->list_node);
1643
1644         /* If this was the last device, remove the services */
1645         if (list_empty(&driver_data.device_list.k_list))
1646                 cryp_algs_unregister_all();
1647
1648         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1649         if (!res_irq)
1650                 dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable",
1651                         __func__);
1652         else {
1653                 disable_irq(res_irq->start);
1654                 free_irq(res_irq->start, device_data);
1655         }
1656
1657         if (cryp_disable_power(&pdev->dev, device_data, false))
1658                 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1659                         __func__);
1660
1661 }
1662
1663 static int ux500_cryp_suspend(struct device *dev)
1664 {
1665         int ret;
1666         struct platform_device *pdev = to_platform_device(dev);
1667         struct cryp_device_data *device_data;
1668         struct resource *res_irq;
1669         struct cryp_ctx *temp_ctx = NULL;
1670
1671         dev_dbg(dev, "[%s]", __func__);
1672
1673         /* Handle state? */
1674         device_data = platform_get_drvdata(pdev);
1675         if (!device_data) {
1676                 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1677                 return -ENOMEM;
1678         }
1679
1680         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1681         if (!res_irq)
1682                 dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__);
1683         else
1684                 disable_irq(res_irq->start);
1685
1686         spin_lock(&device_data->ctx_lock);
1687         if (!device_data->current_ctx)
1688                 device_data->current_ctx++;
1689         spin_unlock(&device_data->ctx_lock);
1690
1691         if (device_data->current_ctx == ++temp_ctx) {
1692                 if (down_interruptible(&driver_data.device_allocation))
1693                         dev_dbg(dev, "[%s]: down_interruptible() failed",
1694                                 __func__);
1695                 ret = cryp_disable_power(dev, device_data, false);
1696
1697         } else
1698                 ret = cryp_disable_power(dev, device_data, true);
1699
1700         if (ret)
1701                 dev_err(dev, "[%s]: cryp_disable_power()", __func__);
1702
1703         return ret;
1704 }
1705
1706 static int ux500_cryp_resume(struct device *dev)
1707 {
1708         int ret = 0;
1709         struct platform_device *pdev = to_platform_device(dev);
1710         struct cryp_device_data *device_data;
1711         struct resource *res_irq;
1712         struct cryp_ctx *temp_ctx = NULL;
1713
1714         dev_dbg(dev, "[%s]", __func__);
1715
1716         device_data = platform_get_drvdata(pdev);
1717         if (!device_data) {
1718                 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1719                 return -ENOMEM;
1720         }
1721
1722         spin_lock(&device_data->ctx_lock);
1723         if (device_data->current_ctx == ++temp_ctx)
1724                 device_data->current_ctx = NULL;
1725         spin_unlock(&device_data->ctx_lock);
1726
1727
1728         if (!device_data->current_ctx)
1729                 up(&driver_data.device_allocation);
1730         else
1731                 ret = cryp_enable_power(dev, device_data, true);
1732
1733         if (ret)
1734                 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1735         else {
1736                 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1737                 if (res_irq)
1738                         enable_irq(res_irq->start);
1739         }
1740
1741         return ret;
1742 }
1743
1744 static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume);
1745
1746 static struct platform_driver cryp_driver = {
1747         .probe  = ux500_cryp_probe,
1748         .remove = ux500_cryp_remove,
1749         .shutdown = ux500_cryp_shutdown,
1750         .driver = {
1751                 .owner = THIS_MODULE,
1752                 .name  = "cryp1",
1753                 .pm    = &ux500_cryp_pm,
1754         }
1755 };
1756
1757 static int __init ux500_cryp_mod_init(void)
1758 {
1759         pr_debug("[%s] is called!", __func__);
1760         klist_init(&driver_data.device_list, NULL, NULL);
1761         /* Initialize the semaphore to 0 devices (locked state) */
1762         sema_init(&driver_data.device_allocation, 0);
1763         return platform_driver_register(&cryp_driver);
1764 }
1765
1766 static void __exit ux500_cryp_mod_fini(void)
1767 {
1768         pr_debug("[%s] is called!", __func__);
1769         platform_driver_unregister(&cryp_driver);
1770         return;
1771 }
1772
1773 module_init(ux500_cryp_mod_init);
1774 module_exit(ux500_cryp_mod_fini);
1775
1776 module_param(cryp_mode, int, 0);
1777
1778 MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 CRYP crypto engine.");
1779 MODULE_ALIAS("aes-all");
1780 MODULE_ALIAS("des-all");
1781
1782 MODULE_LICENSE("GPL");