drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/pci_ids.h>
  17 #include <linux/kthread.h>
  18 #include <linux/sched.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/mutex.h>
  22 #include <linux/delay.h>
  23 #include <linux/ccp.h>
  24 #include <linux/scatterlist.h>
  25 #include <crypto/scatterwalk.h>
  26
  27 #include "ccp-dev.h"
  28
  29
  30 enum ccp_memtype {
  31         CCP_MEMTYPE_SYSTEM = 0,
  32         CCP_MEMTYPE_KSB,
  33         CCP_MEMTYPE_LOCAL,
  34         CCP_MEMTYPE__LAST,
  35 };
  36
  37 struct ccp_dma_info {
  38         dma_addr_t address;
  39         unsigned int offset;
  40         unsigned int length;
  41         enum dma_data_direction dir;
  42 };
  43
  44 struct ccp_dm_workarea {
  45         struct device *dev;
  46         struct dma_pool *dma_pool;
  47         unsigned int length;
  48
  49         u8 *address;
  50         struct ccp_dma_info dma;
  51 };
  52
  53 struct ccp_sg_workarea {
  54         struct scatterlist *sg;
  55         unsigned int nents;
  56         unsigned int length;
  57
  58         struct scatterlist *dma_sg;
  59         struct device *dma_dev;
  60         unsigned int dma_count;
  61         enum dma_data_direction dma_dir;
  62
  63         u32 sg_used;
  64
  65         u32 bytes_left;
  66 };
  67
  68 struct ccp_data {
  69         struct ccp_sg_workarea sg_wa;
  70         struct ccp_dm_workarea dm_wa;
  71 };
  72
  73 struct ccp_mem {
  74         enum ccp_memtype type;
  75         union {
  76                 struct ccp_dma_info dma;
  77                 u32 ksb;
  78         } u;
  79 };
  80
  81 struct ccp_aes_op {
  82         enum ccp_aes_type type;
  83         enum ccp_aes_mode mode;
  84         enum ccp_aes_action action;
  85 };
  86
  87 struct ccp_xts_aes_op {
  88         enum ccp_aes_action action;
  89         enum ccp_xts_aes_unit_size unit_size;
  90 };
  91
  92 struct ccp_sha_op {
  93         enum ccp_sha_type type;
  94         u64 msg_bits;
  95 };
  96
  97 struct ccp_rsa_op {
  98         u32 mod_size;
  99         u32 input_len;
 100 };
 101
 102 struct ccp_passthru_op {
 103         enum ccp_passthru_bitwise bit_mod;
 104         enum ccp_passthru_byteswap byte_swap;
 105 };
 106
 107 struct ccp_ecc_op {
 108         enum ccp_ecc_function function;
 109 };
 110
 111 struct ccp_op {
 112         struct ccp_cmd_queue *cmd_q;
 113
 114         u32 jobid;
 115         u32 ioc;
 116         u32 soc;
 117         u32 ksb_key;
 118         u32 ksb_ctx;
 119         u32 init;
 120         u32 eom;
 121
 122         struct ccp_mem src;
 123         struct ccp_mem dst;
 124
 125         union {
 126                 struct ccp_aes_op aes;
 127                 struct ccp_xts_aes_op xts;
 128                 struct ccp_sha_op sha;
 129                 struct ccp_rsa_op rsa;
 130                 struct ccp_passthru_op passthru;
 131                 struct ccp_ecc_op ecc;
 132         } u;
 133 };
 134
 135 /* The CCP cannot perform zero-length sha operations so the caller
 136  * is required to buffer data for the final operation.  However, a
 137  * sha operation for a message with a total length of zero is valid
 138  * so known values are required to supply the result.
 139  */
 140 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = {
 141         0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
 142         0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
 143         0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00,
 144         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 145 };
 146
 147 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = {
 148         0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9,
 149         0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4,
 150         0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a,
 151         0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00,
 152 };
 153
 154 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = {
 155         0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
 156         0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
 157         0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
 158         0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55,
 159 };
 160
 161 static u32 ccp_addr_lo(struct ccp_dma_info *info)
 162 {
 163         return lower_32_bits(info->address + info->offset);
 164 }
 165
 166 static u32 ccp_addr_hi(struct ccp_dma_info *info)
 167 {
 168         return upper_32_bits(info->address + info->offset) & 0x0000ffff;
 169 }
 170
 171 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
 172 {
 173         struct ccp_cmd_queue *cmd_q = op->cmd_q;
 174         struct ccp_device *ccp = cmd_q->ccp;
 175         void __iomem *cr_addr;
 176         u32 cr0, cmd;
 177         unsigned int i;
 178         int ret = 0;
 179
 180         /* We could read a status register to see how many free slots
 181          * are actually available, but reading that register resets it
 182          * and you could lose some error information.
 183          */
 184         cmd_q->free_slots--;
 185
 186         cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
 187               | (op->jobid << REQ0_JOBID_SHIFT)
 188               | REQ0_WAIT_FOR_WRITE;
 189
 190         if (op->soc)
 191                 cr0 |= REQ0_STOP_ON_COMPLETE
 192                        | REQ0_INT_ON_COMPLETE;
 193
 194         if (op->ioc || !cmd_q->free_slots)
 195                 cr0 |= REQ0_INT_ON_COMPLETE;
 196
 197         /* Start at CMD_REQ1 */
 198         cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
 199
 200         mutex_lock(&ccp->req_mutex);
 201
 202         /* Write CMD_REQ1 through CMD_REQx first */
 203         for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
 204                 iowrite32(*(cr + i), cr_addr);
 205
 206         /* Tell the CCP to start */
 207         wmb();
 208         iowrite32(cr0, ccp->io_regs + CMD_REQ0);
 209
 210         mutex_unlock(&ccp->req_mutex);
 211
 212         if (cr0 & REQ0_INT_ON_COMPLETE) {
 213                 /* Wait for the job to complete */
 214                 ret = wait_event_interruptible(cmd_q->int_queue,
 215                                                cmd_q->int_rcvd);
 216                 if (ret || cmd_q->cmd_error) {
 217                         /* On error delete all related jobs from the queue */
 218                         cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
 219                               | op->jobid;
 220
 221                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 222
 223                         if (!ret)
 224                                 ret = -EIO;
 225                 } else if (op->soc) {
 226                         /* Delete just head job from the queue on SoC */
 227                         cmd = DEL_Q_ACTIVE
 228                               | (cmd_q->id << DEL_Q_ID_SHIFT)
 229                               | op->jobid;
 230
 231                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 232                 }
 233
 234                 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
 235
 236                 cmd_q->int_rcvd = 0;
 237         }
 238
 239         return ret;
 240 }
 241
 242 static int ccp_perform_aes(struct ccp_op *op)
 243 {
 244         u32 cr[6];
 245
 246         /* Fill out the register contents for REQ1 through REQ6 */
 247         cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
 248                 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
 249                 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
 250                 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
 251                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
 252         cr[1] = op->src.u.dma.length - 1;
 253         cr[2] = ccp_addr_lo(&op->src.u.dma);
 254         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 255                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 256                 | ccp_addr_hi(&op->src.u.dma);
 257         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 258         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 259                 | ccp_addr_hi(&op->dst.u.dma);
 260
 261         if (op->u.aes.mode == CCP_AES_MODE_CFB)
 262                 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
 263
 264         if (op->eom)
 265                 cr[0] |= REQ1_EOM;
 266
 267         if (op->init)
 268                 cr[0] |= REQ1_INIT;
 269
 270         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 271 }
 272
 273 static int ccp_perform_xts_aes(struct ccp_op *op)
 274 {
 275         u32 cr[6];
 276
 277         /* Fill out the register contents for REQ1 through REQ6 */
 278         cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
 279                 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
 280                 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
 281                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
 282         cr[1] = op->src.u.dma.length - 1;
 283         cr[2] = ccp_addr_lo(&op->src.u.dma);
 284         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 285                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 286                 | ccp_addr_hi(&op->src.u.dma);
 287         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 288         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 289                 | ccp_addr_hi(&op->dst.u.dma);
 290
 291         if (op->eom)
 292                 cr[0] |= REQ1_EOM;
 293
 294         if (op->init)
 295                 cr[0] |= REQ1_INIT;
 296
 297         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 298 }
 299
 300 static int ccp_perform_sha(struct ccp_op *op)
 301 {
 302         u32 cr[6];
 303
 304         /* Fill out the register contents for REQ1 through REQ6 */
 305         cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
 306                 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
 307                 | REQ1_INIT;
 308         cr[1] = op->src.u.dma.length - 1;
 309         cr[2] = ccp_addr_lo(&op->src.u.dma);
 310         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 311                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 312                 | ccp_addr_hi(&op->src.u.dma);
 313
 314         if (op->eom) {
 315                 cr[0] |= REQ1_EOM;
 316                 cr[4] = lower_32_bits(op->u.sha.msg_bits);
 317                 cr[5] = upper_32_bits(op->u.sha.msg_bits);
 318         } else {
 319                 cr[4] = 0;
 320                 cr[5] = 0;
 321         }
 322
 323         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 324 }
 325
 326 static int ccp_perform_rsa(struct ccp_op *op)
 327 {
 328         u32 cr[6];
 329
 330         /* Fill out the register contents for REQ1 through REQ6 */
 331         cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
 332                 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
 333                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT)
 334                 | REQ1_EOM;
 335         cr[1] = op->u.rsa.input_len - 1;
 336         cr[2] = ccp_addr_lo(&op->src.u.dma);
 337         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 338                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 339                 | ccp_addr_hi(&op->src.u.dma);
 340         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 341         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 342                 | ccp_addr_hi(&op->dst.u.dma);
 343
 344         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 345 }
 346
 347 static int ccp_perform_passthru(struct ccp_op *op)
 348 {
 349         u32 cr[6];
 350
 351         /* Fill out the register contents for REQ1 through REQ6 */
 352         cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
 353                 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
 354                 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
 355
 356         if (op->src.type == CCP_MEMTYPE_SYSTEM)
 357                 cr[1] = op->src.u.dma.length - 1;
 358         else
 359                 cr[1] = op->dst.u.dma.length - 1;
 360
 361         if (op->src.type == CCP_MEMTYPE_SYSTEM) {
 362                 cr[2] = ccp_addr_lo(&op->src.u.dma);
 363                 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 364                         | ccp_addr_hi(&op->src.u.dma);
 365
 366                 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
 367                         cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
 368         } else {
 369                 cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
 370                 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
 371         }
 372
 373         if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
 374                 cr[4] = ccp_addr_lo(&op->dst.u.dma);
 375                 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 376                         | ccp_addr_hi(&op->dst.u.dma);
 377         } else {
 378                 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
 379                 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
 380         }
 381
 382         if (op->eom)
 383                 cr[0] |= REQ1_EOM;
 384
 385         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 386 }
 387
 388 static int ccp_perform_ecc(struct ccp_op *op)
 389 {
 390         u32 cr[6];
 391
 392         /* Fill out the register contents for REQ1 through REQ6 */
 393         cr[0] = REQ1_ECC_AFFINE_CONVERT
 394                 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
 395                 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
 396                 | REQ1_EOM;
 397         cr[1] = op->src.u.dma.length - 1;
 398         cr[2] = ccp_addr_lo(&op->src.u.dma);
 399         cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 400                 | ccp_addr_hi(&op->src.u.dma);
 401         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 402         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 403                 | ccp_addr_hi(&op->dst.u.dma);
 404
 405         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 406 }
 407
 408 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
 409 {
 410         int start;
 411
 412         for (;;) {
 413                 mutex_lock(&ccp->ksb_mutex);
 414
 415                 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
 416                                                         ccp->ksb_count,
 417                                                         ccp->ksb_start,
 418                                                         count, 0);
 419                 if (start <= ccp->ksb_count) {
 420                         bitmap_set(ccp->ksb, start, count);
 421
 422                         mutex_unlock(&ccp->ksb_mutex);
 423                         break;
 424                 }
 425
 426                 ccp->ksb_avail = 0;
 427
 428                 mutex_unlock(&ccp->ksb_mutex);
 429
 430                 /* Wait for KSB entries to become available */
 431                 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
 432                         return 0;
 433         }
 434
 435         return KSB_START + start;
 436 }
 437
 438 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
 439                          unsigned int count)
 440 {
 441         if (!start)
 442                 return;
 443
 444         mutex_lock(&ccp->ksb_mutex);
 445
 446         bitmap_clear(ccp->ksb, start - KSB_START, count);
 447
 448         ccp->ksb_avail = 1;
 449
 450         mutex_unlock(&ccp->ksb_mutex);
 451
 452         wake_up_interruptible_all(&ccp->ksb_queue);
 453 }
 454
 455 static u32 ccp_gen_jobid(struct ccp_device *ccp)
 456 {
 457         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
 458 }
 459
 460 static void ccp_sg_free(struct ccp_sg_workarea *wa)
 461 {
 462         if (wa->dma_count)
 463                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
 464
 465         wa->dma_count = 0;
 466 }
 467
 468 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
 469                                 struct scatterlist *sg, unsigned int len,
 470                                 enum dma_data_direction dma_dir)
 471 {
 472         memset(wa, 0, sizeof(*wa));
 473
 474         wa->sg = sg;
 475         if (!sg)
 476                 return 0;
 477
 478         wa->nents = sg_nents(sg);
 479         wa->length = sg->length;
 480         wa->bytes_left = len;
 481         wa->sg_used = 0;
 482
 483         if (len == 0)
 484                 return 0;
 485
 486         if (dma_dir == DMA_NONE)
 487                 return 0;
 488
 489         wa->dma_sg = sg;
 490         wa->dma_dev = dev;
 491         wa->dma_dir = dma_dir;
 492         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 493         if (!wa->dma_count)
 494                 return -ENOMEM;
 495
 496
 497         return 0;
 498 }
 499
 500 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 501 {
 502         unsigned int nbytes = min(len, wa->bytes_left);
 503
 504         if (!wa->sg)
 505                 return;
 506
 507         wa->sg_used += nbytes;
 508         wa->bytes_left -= nbytes;
 509         if (wa->sg_used == wa->sg->length) {
 510                 wa->sg = sg_next(wa->sg);
 511                 wa->sg_used = 0;
 512         }
 513 }
 514
 515 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 516 {
 517         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 518                 if (wa->address)
 519                         dma_pool_free(wa->dma_pool, wa->address,
 520                                       wa->dma.address);
 521         } else {
 522                 if (wa->dma.address)
 523                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 524                                          wa->dma.dir);
 525                 kfree(wa->address);
 526         }
 527
 528         wa->address = NULL;
 529         wa->dma.address = 0;
 530 }
 531
 532 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 533                                 struct ccp_cmd_queue *cmd_q,
 534                                 unsigned int len,
 535                                 enum dma_data_direction dir)
 536 {
 537         memset(wa, 0, sizeof(*wa));
 538
 539         if (!len)
 540                 return 0;
 541
 542         wa->dev = cmd_q->ccp->dev;
 543         wa->length = len;
 544
 545         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 546                 wa->dma_pool = cmd_q->dma_pool;
 547
 548                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 549                                              &wa->dma.address);
 550                 if (!wa->address)
 551                         return -ENOMEM;
 552
 553                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 554
 555                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 556         } else {
 557                 wa->address = kzalloc(len, GFP_KERNEL);
 558                 if (!wa->address)
 559                         return -ENOMEM;
 560
 561                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 562                                                  dir);
 563                 if (!wa->dma.address)
 564                         return -ENOMEM;
 565
 566                 wa->dma.length = len;
 567         }
 568         wa->dma.dir = dir;
 569
 570         return 0;
 571 }
 572
 573 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 574                             struct scatterlist *sg, unsigned int sg_offset,
 575                             unsigned int len)
 576 {
 577         WARN_ON(!wa->address);
 578
 579         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 580                                  0);
 581 }
 582
 583 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 584                             struct scatterlist *sg, unsigned int sg_offset,
 585                             unsigned int len)
 586 {
 587         WARN_ON(!wa->address);
 588
 589         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 590                                  1);
 591 }
 592
 593 static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 594                                     struct scatterlist *sg,
 595                                     unsigned int len, unsigned int se_len,
 596                                     bool sign_extend)
 597 {
 598         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 599         u8 buffer[CCP_REVERSE_BUF_SIZE];
 600
 601         BUG_ON(se_len > sizeof(buffer));
 602
 603         sg_offset = len;
 604         dm_offset = 0;
 605         nbytes = len;
 606         while (nbytes) {
 607                 ksb_len = min_t(unsigned int, nbytes, se_len);
 608                 sg_offset -= ksb_len;
 609
 610                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
 611                 for (i = 0; i < ksb_len; i++)
 612                         wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
 613
 614                 dm_offset += ksb_len;
 615                 nbytes -= ksb_len;
 616
 617                 if ((ksb_len != se_len) && sign_extend) {
 618                         /* Must sign-extend to nearest sign-extend length */
 619                         if (wa->address[dm_offset - 1] & 0x80)
 620                                 memset(wa->address + dm_offset, 0xff,
 621                                        se_len - ksb_len);
 622                 }
 623         }
 624 }
 625
 626 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 627                                     struct scatterlist *sg,
 628                                     unsigned int len)
 629 {
 630         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 631         u8 buffer[CCP_REVERSE_BUF_SIZE];
 632
 633         sg_offset = 0;
 634         dm_offset = len;
 635         nbytes = len;
 636         while (nbytes) {
 637                 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
 638                 dm_offset -= ksb_len;
 639
 640                 for (i = 0; i < ksb_len; i++)
 641                         buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
 642                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
 643
 644                 sg_offset += ksb_len;
 645                 nbytes -= ksb_len;
 646         }
 647 }
 648
 649 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 650 {
 651         ccp_dm_free(&data->dm_wa);
 652         ccp_sg_free(&data->sg_wa);
 653 }
 654
 655 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 656                          struct scatterlist *sg, unsigned int sg_len,
 657                          unsigned int dm_len,
 658                          enum dma_data_direction dir)
 659 {
 660         int ret;
 661
 662         memset(data, 0, sizeof(*data));
 663
 664         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 665                                    dir);
 666         if (ret)
 667                 goto e_err;
 668
 669         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 670         if (ret)
 671                 goto e_err;
 672
 673         return 0;
 674
 675 e_err:
 676         ccp_free_data(data, cmd_q);
 677
 678         return ret;
 679 }
 680
 681 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 682 {
 683         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 684         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 685         unsigned int buf_count, nbytes;
 686
 687         /* Clear the buffer if setting it */
 688         if (!from)
 689                 memset(dm_wa->address, 0, dm_wa->length);
 690
 691         if (!sg_wa->sg)
 692                 return 0;
 693
 694         /* Perform the copy operation */
 695         nbytes = min(sg_wa->bytes_left, dm_wa->length);
 696         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 697                                  nbytes, from);
 698
 699         /* Update the structures and generate the count */
 700         buf_count = 0;
 701         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 702                 nbytes = min3(sg_wa->sg->length - sg_wa->sg_used,
 703                               dm_wa->length - buf_count,
 704                               sg_wa->bytes_left);
 705
 706                 buf_count += nbytes;
 707                 ccp_update_sg_workarea(sg_wa, nbytes);
 708         }
 709
 710         return buf_count;
 711 }
 712
 713 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 714 {
 715         return ccp_queue_buf(data, 0);
 716 }
 717
 718 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 719 {
 720         return ccp_queue_buf(data, 1);
 721 }
 722
 723 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 724                              struct ccp_op *op, unsigned int block_size,
 725                              bool blocksize_op)
 726 {
 727         unsigned int sg_src_len, sg_dst_len, op_len;
 728
 729         /* The CCP can only DMA from/to one address each per operation. This
 730          * requires that we find the smallest DMA area between the source
 731          * and destination.
 732          */
 733         sg_src_len = min(sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used,
 734                          src->sg_wa.bytes_left);
 735
 736         if (dst) {
 737                 sg_dst_len = min(sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used,
 738                                  src->sg_wa.bytes_left);
 739                 op_len = min(sg_src_len, sg_dst_len);
 740         } else
 741                 op_len = sg_src_len;
 742
 743         /* The data operation length will be at least block_size in length
 744          * or the smaller of available sg room remaining for the source or
 745          * the destination
 746          */
 747         op_len = max(op_len, block_size);
 748
 749         /* Unless we have to buffer data, there's no reason to wait */
 750         op->soc = 0;
 751
 752         if (sg_src_len < block_size) {
 753                 /* Not enough data in the sg element, so it
 754                  * needs to be buffered into a blocksize chunk
 755                  */
 756                 int cp_len = ccp_fill_queue_buf(src);
 757
 758                 op->soc = 1;
 759                 op->src.u.dma.address = src->dm_wa.dma.address;
 760                 op->src.u.dma.offset = 0;
 761                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 762         } else {
 763                 /* Enough data in the sg element, but we need to
 764                  * adjust for any previously copied data
 765                  */
 766                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 767                 op->src.u.dma.offset = src->sg_wa.sg_used;
 768                 op->src.u.dma.length = op_len & ~(block_size - 1);
 769
 770                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 771         }
 772
 773         if (dst) {
 774                 if (sg_dst_len < block_size) {
 775                         /* Not enough room in the sg element or we're on the
 776                          * last piece of data (when using padding), so the
 777                          * output needs to be buffered into a blocksize chunk
 778                          */
 779                         op->soc = 1;
 780                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 781                         op->dst.u.dma.offset = 0;
 782                         op->dst.u.dma.length = op->src.u.dma.length;
 783                 } else {
 784                         /* Enough room in the sg element, but we need to
 785                          * adjust for any previously used area
 786                          */
 787                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 788                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 789                         op->dst.u.dma.length = op->src.u.dma.length;
 790                 }
 791         }
 792 }
 793
 794 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 795                              struct ccp_op *op)
 796 {
 797         op->init = 0;
 798
 799         if (dst) {
 800                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 801                         ccp_empty_queue_buf(dst);
 802                 else
 803                         ccp_update_sg_workarea(&dst->sg_wa,
 804                                                op->dst.u.dma.length);
 805         }
 806 }
 807
 808 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
 809                                 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 810                                 u32 byte_swap, bool from)
 811 {
 812         struct ccp_op op;
 813
 814         memset(&op, 0, sizeof(op));
 815
 816         op.cmd_q = cmd_q;
 817         op.jobid = jobid;
 818         op.eom = 1;
 819
 820         if (from) {
 821                 op.soc = 1;
 822                 op.src.type = CCP_MEMTYPE_KSB;
 823                 op.src.u.ksb = ksb;
 824                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 825                 op.dst.u.dma.address = wa->dma.address;
 826                 op.dst.u.dma.length = wa->length;
 827         } else {
 828                 op.src.type = CCP_MEMTYPE_SYSTEM;
 829                 op.src.u.dma.address = wa->dma.address;
 830                 op.src.u.dma.length = wa->length;
 831                 op.dst.type = CCP_MEMTYPE_KSB;
 832                 op.dst.u.ksb = ksb;
 833         }
 834
 835         op.u.passthru.byte_swap = byte_swap;
 836
 837         return ccp_perform_passthru(&op);
 838 }
 839
 840 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
 841                            struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 842                            u32 byte_swap)
 843 {
 844         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
 845 }
 846
 847 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
 848                              struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 849                              u32 byte_swap)
 850 {
 851         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
 852 }
 853
 854 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 855                                 struct ccp_cmd *cmd)
 856 {
 857         struct ccp_aes_engine *aes = &cmd->u.aes;
 858         struct ccp_dm_workarea key, ctx;
 859         struct ccp_data src;
 860         struct ccp_op op;
 861         unsigned int dm_offset;
 862         int ret;
 863
 864         if (!((aes->key_len == AES_KEYSIZE_128) ||
 865               (aes->key_len == AES_KEYSIZE_192) ||
 866               (aes->key_len == AES_KEYSIZE_256)))
 867                 return -EINVAL;
 868
 869         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 870                 return -EINVAL;
 871
 872         if (aes->iv_len != AES_BLOCK_SIZE)
 873                 return -EINVAL;
 874
 875         if (!aes->key || !aes->iv || !aes->src)
 876                 return -EINVAL;
 877
 878         if (aes->cmac_final) {
 879                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 880                         return -EINVAL;
 881
 882                 if (!aes->cmac_key)
 883                         return -EINVAL;
 884         }
 885
 886         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
 887         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
 888
 889         ret = -EIO;
 890         memset(&op, 0, sizeof(op));
 891         op.cmd_q = cmd_q;
 892         op.jobid = ccp_gen_jobid(cmd_q->ccp);
 893         op.ksb_key = cmd_q->ksb_key;
 894         op.ksb_ctx = cmd_q->ksb_ctx;
 895         op.init = 1;
 896         op.u.aes.type = aes->type;
 897         op.u.aes.mode = aes->mode;
 898         op.u.aes.action = aes->action;
 899
 900         /* All supported key sizes fit in a single (32-byte) KSB entry
 901          * and must be in little endian format. Use the 256-bit byte
 902          * swap passthru option to convert from big endian to little
 903          * endian.
 904          */
 905         ret = ccp_init_dm_workarea(&key, cmd_q,
 906                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
 907                                    DMA_TO_DEVICE);
 908         if (ret)
 909                 return ret;
 910
 911         dm_offset = CCP_KSB_BYTES - aes->key_len;
 912         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 913         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
 914                               CCP_PASSTHRU_BYTESWAP_256BIT);
 915         if (ret) {
 916                 cmd->engine_error = cmd_q->cmd_error;
 917                 goto e_key;
 918         }
 919
 920         /* The AES context fits in a single (32-byte) KSB entry and
 921          * must be in little endian format. Use the 256-bit byte swap
 922          * passthru option to convert from big endian to little endian.
 923          */
 924         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 925                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
 926                                    DMA_BIDIRECTIONAL);
 927         if (ret)
 928                 goto e_key;
 929
 930         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 931         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 932         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 933                               CCP_PASSTHRU_BYTESWAP_256BIT);
 934         if (ret) {
 935                 cmd->engine_error = cmd_q->cmd_error;
 936                 goto e_ctx;
 937         }
 938
 939         /* Send data to the CCP AES engine */
 940         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 941                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 942         if (ret)
 943                 goto e_ctx;
 944
 945         while (src.sg_wa.bytes_left) {
 946                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 947                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 948                         op.eom = 1;
 949
 950                         /* Push the K1/K2 key to the CCP now */
 951                         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
 952                                                 op.ksb_ctx,
 953                                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 954                         if (ret) {
 955                                 cmd->engine_error = cmd_q->cmd_error;
 956                                 goto e_src;
 957                         }
 958
 959                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 960                                         aes->cmac_key_len);
 961                         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 962                                               CCP_PASSTHRU_BYTESWAP_256BIT);
 963                         if (ret) {
 964                                 cmd->engine_error = cmd_q->cmd_error;
 965                                 goto e_src;
 966                         }
 967                 }
 968
 969                 ret = ccp_perform_aes(&op);
 970                 if (ret) {
 971                         cmd->engine_error = cmd_q->cmd_error;
 972                         goto e_src;
 973                 }
 974
 975                 ccp_process_data(&src, NULL, &op);
 976         }
 977
 978         /* Retrieve the AES context - convert from LE to BE using
 979          * 32-byte (256-bit) byteswapping
 980          */
 981         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 982                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 983         if (ret) {
 984                 cmd->engine_error = cmd_q->cmd_error;
 985                 goto e_src;
 986         }
 987
 988         /* ...but we only need AES_BLOCK_SIZE bytes */
 989         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 990         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 991
 992 e_src:
 993         ccp_free_data(&src, cmd_q);
 994
 995 e_ctx:
 996         ccp_dm_free(&ctx);
 997
 998 e_key:
 999         ccp_dm_free(&key);
1000
1001         return ret;
1002 }
1003
1004 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1005 {
1006         struct ccp_aes_engine *aes = &cmd->u.aes;
1007         struct ccp_dm_workarea key, ctx;
1008         struct ccp_data src, dst;
1009         struct ccp_op op;
1010         unsigned int dm_offset;
1011         bool in_place = false;
1012         int ret;
1013
1014         if (aes->mode == CCP_AES_MODE_CMAC)
1015                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
1016
1017         if (!((aes->key_len == AES_KEYSIZE_128) ||
1018               (aes->key_len == AES_KEYSIZE_192) ||
1019               (aes->key_len == AES_KEYSIZE_256)))
1020                 return -EINVAL;
1021
1022         if (((aes->mode == CCP_AES_MODE_ECB) ||
1023              (aes->mode == CCP_AES_MODE_CBC) ||
1024              (aes->mode == CCP_AES_MODE_CFB)) &&
1025             (aes->src_len & (AES_BLOCK_SIZE - 1)))
1026                 return -EINVAL;
1027
1028         if (!aes->key || !aes->src || !aes->dst)
1029                 return -EINVAL;
1030
1031         if (aes->mode != CCP_AES_MODE_ECB) {
1032                 if (aes->iv_len != AES_BLOCK_SIZE)
1033                         return -EINVAL;
1034
1035                 if (!aes->iv)
1036                         return -EINVAL;
1037         }
1038
1039         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
1040         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
1041
1042         ret = -EIO;
1043         memset(&op, 0, sizeof(op));
1044         op.cmd_q = cmd_q;
1045         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1046         op.ksb_key = cmd_q->ksb_key;
1047         op.ksb_ctx = cmd_q->ksb_ctx;
1048         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
1049         op.u.aes.type = aes->type;
1050         op.u.aes.mode = aes->mode;
1051         op.u.aes.action = aes->action;
1052
1053         /* All supported key sizes fit in a single (32-byte) KSB entry
1054          * and must be in little endian format. Use the 256-bit byte
1055          * swap passthru option to convert from big endian to little
1056          * endian.
1057          */
1058         ret = ccp_init_dm_workarea(&key, cmd_q,
1059                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1060                                    DMA_TO_DEVICE);
1061         if (ret)
1062                 return ret;
1063
1064         dm_offset = CCP_KSB_BYTES - aes->key_len;
1065         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
1066         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1067                               CCP_PASSTHRU_BYTESWAP_256BIT);
1068         if (ret) {
1069                 cmd->engine_error = cmd_q->cmd_error;
1070                 goto e_key;
1071         }
1072
1073         /* The AES context fits in a single (32-byte) KSB entry and
1074          * must be in little endian format. Use the 256-bit byte swap
1075          * passthru option to convert from big endian to little endian.
1076          */
1077         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1078                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1079                                    DMA_BIDIRECTIONAL);
1080         if (ret)
1081                 goto e_key;
1082
1083         if (aes->mode != CCP_AES_MODE_ECB) {
1084                 /* Load the AES context - conver to LE */
1085                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1086                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1087                 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1088                                       CCP_PASSTHRU_BYTESWAP_256BIT);
1089                 if (ret) {
1090                         cmd->engine_error = cmd_q->cmd_error;
1091                         goto e_ctx;
1092                 }
1093         }
1094
1095         /* Prepare the input and output data workareas. For in-place
1096          * operations we need to set the dma direction to BIDIRECTIONAL
1097          * and copy the src workarea to the dst workarea.
1098          */
1099         if (sg_virt(aes->src) == sg_virt(aes->dst))
1100                 in_place = true;
1101
1102         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1103                             AES_BLOCK_SIZE,
1104                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1105         if (ret)
1106                 goto e_ctx;
1107
1108         if (in_place)
1109                 dst = src;
1110         else {
1111                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1112                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1113                 if (ret)
1114                         goto e_src;
1115         }
1116
1117         /* Send data to the CCP AES engine */
1118         while (src.sg_wa.bytes_left) {
1119                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1120                 if (!src.sg_wa.bytes_left) {
1121                         op.eom = 1;
1122
1123                         /* Since we don't retrieve the AES context in ECB
1124                          * mode we have to wait for the operation to complete
1125                          * on the last piece of data
1126                          */
1127                         if (aes->mode == CCP_AES_MODE_ECB)
1128                                 op.soc = 1;
1129                 }
1130
1131                 ret = ccp_perform_aes(&op);
1132                 if (ret) {
1133                         cmd->engine_error = cmd_q->cmd_error;
1134                         goto e_dst;
1135                 }
1136
1137                 ccp_process_data(&src, &dst, &op);
1138         }
1139
1140         if (aes->mode != CCP_AES_MODE_ECB) {
1141                 /* Retrieve the AES context - convert from LE to BE using
1142                  * 32-byte (256-bit) byteswapping
1143                  */
1144                 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1145                                         CCP_PASSTHRU_BYTESWAP_256BIT);
1146                 if (ret) {
1147                         cmd->engine_error = cmd_q->cmd_error;
1148                         goto e_dst;
1149                 }
1150
1151                 /* ...but we only need AES_BLOCK_SIZE bytes */
1152                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1153                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1154         }
1155
1156 e_dst:
1157         if (!in_place)
1158                 ccp_free_data(&dst, cmd_q);
1159
1160 e_src:
1161         ccp_free_data(&src, cmd_q);
1162
1163 e_ctx:
1164         ccp_dm_free(&ctx);
1165
1166 e_key:
1167         ccp_dm_free(&key);
1168
1169         return ret;
1170 }
1171
1172 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1173                                struct ccp_cmd *cmd)
1174 {
1175         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1176         struct ccp_dm_workarea key, ctx;
1177         struct ccp_data src, dst;
1178         struct ccp_op op;
1179         unsigned int unit_size, dm_offset;
1180         bool in_place = false;
1181         int ret;
1182
1183         switch (xts->unit_size) {
1184         case CCP_XTS_AES_UNIT_SIZE_16:
1185                 unit_size = 16;
1186                 break;
1187         case CCP_XTS_AES_UNIT_SIZE_512:
1188                 unit_size = 512;
1189                 break;
1190         case CCP_XTS_AES_UNIT_SIZE_1024:
1191                 unit_size = 1024;
1192                 break;
1193         case CCP_XTS_AES_UNIT_SIZE_2048:
1194                 unit_size = 2048;
1195                 break;
1196         case CCP_XTS_AES_UNIT_SIZE_4096:
1197                 unit_size = 4096;
1198                 break;
1199
1200         default:
1201                 return -EINVAL;
1202         }
1203
1204         if (xts->key_len != AES_KEYSIZE_128)
1205                 return -EINVAL;
1206
1207         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1208                 return -EINVAL;
1209
1210         if (xts->iv_len != AES_BLOCK_SIZE)
1211                 return -EINVAL;
1212
1213         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1214                 return -EINVAL;
1215
1216         BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
1217         BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
1218
1219         ret = -EIO;
1220         memset(&op, 0, sizeof(op));
1221         op.cmd_q = cmd_q;
1222         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1223         op.ksb_key = cmd_q->ksb_key;
1224         op.ksb_ctx = cmd_q->ksb_ctx;
1225         op.init = 1;
1226         op.u.xts.action = xts->action;
1227         op.u.xts.unit_size = xts->unit_size;
1228
1229         /* All supported key sizes fit in a single (32-byte) KSB entry
1230          * and must be in little endian format. Use the 256-bit byte
1231          * swap passthru option to convert from big endian to little
1232          * endian.
1233          */
1234         ret = ccp_init_dm_workarea(&key, cmd_q,
1235                                    CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1236                                    DMA_TO_DEVICE);
1237         if (ret)
1238                 return ret;
1239
1240         dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
1241         ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1242         ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1243         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1244                               CCP_PASSTHRU_BYTESWAP_256BIT);
1245         if (ret) {
1246                 cmd->engine_error = cmd_q->cmd_error;
1247                 goto e_key;
1248         }
1249
1250         /* The AES context fits in a single (32-byte) KSB entry and
1251          * for XTS is already in little endian format so no byte swapping
1252          * is needed.
1253          */
1254         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1255                                    CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1256                                    DMA_BIDIRECTIONAL);
1257         if (ret)
1258                 goto e_key;
1259
1260         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1261         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1262                               CCP_PASSTHRU_BYTESWAP_NOOP);
1263         if (ret) {
1264                 cmd->engine_error = cmd_q->cmd_error;
1265                 goto e_ctx;
1266         }
1267
1268         /* Prepare the input and output data workareas. For in-place
1269          * operations we need to set the dma direction to BIDIRECTIONAL
1270          * and copy the src workarea to the dst workarea.
1271          */
1272         if (sg_virt(xts->src) == sg_virt(xts->dst))
1273                 in_place = true;
1274
1275         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1276                             unit_size,
1277                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1278         if (ret)
1279                 goto e_ctx;
1280
1281         if (in_place)
1282                 dst = src;
1283         else {
1284                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1285                                     unit_size, DMA_FROM_DEVICE);
1286                 if (ret)
1287                         goto e_src;
1288         }
1289
1290         /* Send data to the CCP AES engine */
1291         while (src.sg_wa.bytes_left) {
1292                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1293                 if (!src.sg_wa.bytes_left)
1294                         op.eom = 1;
1295
1296                 ret = ccp_perform_xts_aes(&op);
1297                 if (ret) {
1298                         cmd->engine_error = cmd_q->cmd_error;
1299                         goto e_dst;
1300                 }
1301
1302                 ccp_process_data(&src, &dst, &op);
1303         }
1304
1305         /* Retrieve the AES context - convert from LE to BE using
1306          * 32-byte (256-bit) byteswapping
1307          */
1308         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1309                                 CCP_PASSTHRU_BYTESWAP_256BIT);
1310         if (ret) {
1311                 cmd->engine_error = cmd_q->cmd_error;
1312                 goto e_dst;
1313         }
1314
1315         /* ...but we only need AES_BLOCK_SIZE bytes */
1316         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1317         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1318
1319 e_dst:
1320         if (!in_place)
1321                 ccp_free_data(&dst, cmd_q);
1322
1323 e_src:
1324         ccp_free_data(&src, cmd_q);
1325
1326 e_ctx:
1327         ccp_dm_free(&ctx);
1328
1329 e_key:
1330         ccp_dm_free(&key);
1331
1332         return ret;
1333 }
1334
1335 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1336 {
1337         struct ccp_sha_engine *sha = &cmd->u.sha;
1338         struct ccp_dm_workarea ctx;
1339         struct ccp_data src;
1340         struct ccp_op op;
1341         int ret;
1342
1343         if (sha->ctx_len != CCP_SHA_CTXSIZE)
1344                 return -EINVAL;
1345
1346         if (!sha->ctx)
1347                 return -EINVAL;
1348
1349         if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
1350                 return -EINVAL;
1351
1352         if (!sha->src_len) {
1353                 const u8 *sha_zero;
1354
1355                 /* Not final, just return */
1356                 if (!sha->final)
1357                         return 0;
1358
1359                 /* CCP can't do a zero length sha operation so the caller
1360                  * must buffer the data.
1361                  */
1362                 if (sha->msg_bits)
1363                         return -EINVAL;
1364
1365                 /* A sha operation for a message with a total length of zero,
1366                  * return known result.
1367                  */
1368                 switch (sha->type) {
1369                 case CCP_SHA_TYPE_1:
1370                         sha_zero = ccp_sha1_zero;
1371                         break;
1372                 case CCP_SHA_TYPE_224:
1373                         sha_zero = ccp_sha224_zero;
1374                         break;
1375                 case CCP_SHA_TYPE_256:
1376                         sha_zero = ccp_sha256_zero;
1377                         break;
1378                 default:
1379                         return -EINVAL;
1380                 }
1381
1382                 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1383                                          sha->ctx_len, 1);
1384
1385                 return 0;
1386         }
1387
1388         if (!sha->src)
1389                 return -EINVAL;
1390
1391         BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1392
1393         memset(&op, 0, sizeof(op));
1394         op.cmd_q = cmd_q;
1395         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1396         op.ksb_ctx = cmd_q->ksb_ctx;
1397         op.u.sha.type = sha->type;
1398         op.u.sha.msg_bits = sha->msg_bits;
1399
1400         /* The SHA context fits in a single (32-byte) KSB entry and
1401          * must be in little endian format. Use the 256-bit byte swap
1402          * passthru option to convert from big endian to little endian.
1403          */
1404         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1405                                    CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1406                                    DMA_BIDIRECTIONAL);
1407         if (ret)
1408                 return ret;
1409
1410         ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1411         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1412                               CCP_PASSTHRU_BYTESWAP_256BIT);
1413         if (ret) {
1414                 cmd->engine_error = cmd_q->cmd_error;
1415                 goto e_ctx;
1416         }
1417
1418         /* Send data to the CCP SHA engine */
1419         ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1420                             CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1421         if (ret)
1422                 goto e_ctx;
1423
1424         while (src.sg_wa.bytes_left) {
1425                 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1426                 if (sha->final && !src.sg_wa.bytes_left)
1427                         op.eom = 1;
1428
1429                 ret = ccp_perform_sha(&op);
1430                 if (ret) {
1431                         cmd->engine_error = cmd_q->cmd_error;
1432                         goto e_data;
1433                 }
1434
1435                 ccp_process_data(&src, NULL, &op);
1436         }
1437
1438         /* Retrieve the SHA context - convert from LE to BE using
1439          * 32-byte (256-bit) byteswapping to BE
1440          */
1441         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1442                                 CCP_PASSTHRU_BYTESWAP_256BIT);
1443         if (ret) {
1444                 cmd->engine_error = cmd_q->cmd_error;
1445                 goto e_data;
1446         }
1447
1448         ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1449
1450 e_data:
1451         ccp_free_data(&src, cmd_q);
1452
1453 e_ctx:
1454         ccp_dm_free(&ctx);
1455
1456         return ret;
1457 }
1458
1459 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1460 {
1461         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1462         struct ccp_dm_workarea exp, src;
1463         struct ccp_data dst;
1464         struct ccp_op op;
1465         unsigned int ksb_count, i_len, o_len;
1466         int ret;
1467
1468         if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1469                 return -EINVAL;
1470
1471         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1472                 return -EINVAL;
1473
1474         /* The RSA modulus must precede the message being acted upon, so
1475          * it must be copied to a DMA area where the message and the
1476          * modulus can be concatenated.  Therefore the input buffer
1477          * length required is twice the output buffer length (which
1478          * must be a multiple of 256-bits).
1479          */
1480         o_len = ((rsa->key_size + 255) / 256) * 32;
1481         i_len = o_len * 2;
1482
1483         ksb_count = o_len / CCP_KSB_BYTES;
1484
1485         memset(&op, 0, sizeof(op));
1486         op.cmd_q = cmd_q;
1487         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1488         op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1489         if (!op.ksb_key)
1490                 return -EIO;
1491
1492         /* The RSA exponent may span multiple (32-byte) KSB entries and must
1493          * be in little endian format. Reverse copy each 32-byte chunk
1494          * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1495          * and each byte within that chunk and do not perform any byte swap
1496          * operations on the passthru operation.
1497          */
1498         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1499         if (ret)
1500                 goto e_ksb;
1501
1502         ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES,
1503                                 true);
1504         ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1505                               CCP_PASSTHRU_BYTESWAP_NOOP);
1506         if (ret) {
1507                 cmd->engine_error = cmd_q->cmd_error;
1508                 goto e_exp;
1509         }
1510
1511         /* Concatenate the modulus and the message. Both the modulus and
1512          * the operands must be in little endian format.  Since the input
1513          * is in big endian format it must be converted.
1514          */
1515         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1516         if (ret)
1517                 goto e_exp;
1518
1519         ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES,
1520                                 true);
1521         src.address += o_len;   /* Adjust the address for the copy operation */
1522         ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES,
1523                                 true);
1524         src.address -= o_len;   /* Reset the address to original value */
1525
1526         /* Prepare the output area for the operation */
1527         ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1528                             o_len, DMA_FROM_DEVICE);
1529         if (ret)
1530                 goto e_src;
1531
1532         op.soc = 1;
1533         op.src.u.dma.address = src.dma.address;
1534         op.src.u.dma.offset = 0;
1535         op.src.u.dma.length = i_len;
1536         op.dst.u.dma.address = dst.dm_wa.dma.address;
1537         op.dst.u.dma.offset = 0;
1538         op.dst.u.dma.length = o_len;
1539
1540         op.u.rsa.mod_size = rsa->key_size;
1541         op.u.rsa.input_len = i_len;
1542
1543         ret = ccp_perform_rsa(&op);
1544         if (ret) {
1545                 cmd->engine_error = cmd_q->cmd_error;
1546                 goto e_dst;
1547         }
1548
1549         ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1550
1551 e_dst:
1552         ccp_free_data(&dst, cmd_q);
1553
1554 e_src:
1555         ccp_dm_free(&src);
1556
1557 e_exp:
1558         ccp_dm_free(&exp);
1559
1560 e_ksb:
1561         ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1562
1563         return ret;
1564 }
1565
1566 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1567                                 struct ccp_cmd *cmd)
1568 {
1569         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1570         struct ccp_dm_workarea mask;
1571         struct ccp_data src, dst;
1572         struct ccp_op op;
1573         bool in_place = false;
1574         unsigned int i;
1575         int ret;
1576
1577         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1578                 return -EINVAL;
1579
1580         if (!pt->src || !pt->dst)
1581                 return -EINVAL;
1582
1583         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1584                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1585                         return -EINVAL;
1586                 if (!pt->mask)
1587                         return -EINVAL;
1588         }
1589
1590         BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1591
1592         memset(&op, 0, sizeof(op));
1593         op.cmd_q = cmd_q;
1594         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1595
1596         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1597                 /* Load the mask */
1598                 op.ksb_key = cmd_q->ksb_key;
1599
1600                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1601                                            CCP_PASSTHRU_KSB_COUNT *
1602                                            CCP_KSB_BYTES,
1603                                            DMA_TO_DEVICE);
1604                 if (ret)
1605                         return ret;
1606
1607                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1608                 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1609                                       CCP_PASSTHRU_BYTESWAP_NOOP);
1610                 if (ret) {
1611                         cmd->engine_error = cmd_q->cmd_error;
1612                         goto e_mask;
1613                 }
1614         }
1615
1616         /* Prepare the input and output data workareas. For in-place
1617          * operations we need to set the dma direction to BIDIRECTIONAL
1618          * and copy the src workarea to the dst workarea.
1619          */
1620         if (sg_virt(pt->src) == sg_virt(pt->dst))
1621                 in_place = true;
1622
1623         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1624                             CCP_PASSTHRU_MASKSIZE,
1625                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1626         if (ret)
1627                 goto e_mask;
1628
1629         if (in_place)
1630                 dst = src;
1631         else {
1632                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1633                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1634                 if (ret)
1635                         goto e_src;
1636         }
1637
1638         /* Send data to the CCP Passthru engine
1639          *   Because the CCP engine works on a single source and destination
1640          *   dma address at a time, each entry in the source scatterlist
1641          *   (after the dma_map_sg call) must be less than or equal to the
1642          *   (remaining) length in the destination scatterlist entry and the
1643          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1644          */
1645         dst.sg_wa.sg_used = 0;
1646         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1647                 if (!dst.sg_wa.sg ||
1648                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1649                         ret = -EINVAL;
1650                         goto e_dst;
1651                 }
1652
1653                 if (i == src.sg_wa.dma_count) {
1654                         op.eom = 1;
1655                         op.soc = 1;
1656                 }
1657
1658                 op.src.type = CCP_MEMTYPE_SYSTEM;
1659                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1660                 op.src.u.dma.offset = 0;
1661                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1662
1663                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1664                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1665                 op.src.u.dma.offset = dst.sg_wa.sg_used;
1666                 op.src.u.dma.length = op.src.u.dma.length;
1667
1668                 ret = ccp_perform_passthru(&op);
1669                 if (ret) {
1670                         cmd->engine_error = cmd_q->cmd_error;
1671                         goto e_dst;
1672                 }
1673
1674                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1675                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1676                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1677                         dst.sg_wa.sg_used = 0;
1678                 }
1679                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1680         }
1681
1682 e_dst:
1683         if (!in_place)
1684                 ccp_free_data(&dst, cmd_q);
1685
1686 e_src:
1687         ccp_free_data(&src, cmd_q);
1688
1689 e_mask:
1690         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1691                 ccp_dm_free(&mask);
1692
1693         return ret;
1694 }
1695
1696 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1697 {
1698         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1699         struct ccp_dm_workarea src, dst;
1700         struct ccp_op op;
1701         int ret;
1702         u8 *save;
1703
1704         if (!ecc->u.mm.operand_1 ||
1705             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1706                 return -EINVAL;
1707
1708         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1709                 if (!ecc->u.mm.operand_2 ||
1710                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1711                         return -EINVAL;
1712
1713         if (!ecc->u.mm.result ||
1714             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1715                 return -EINVAL;
1716
1717         memset(&op, 0, sizeof(op));
1718         op.cmd_q = cmd_q;
1719         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1720
1721         /* Concatenate the modulus and the operands. Both the modulus and
1722          * the operands must be in little endian format.  Since the input
1723          * is in big endian format it must be converted and placed in a
1724          * fixed length buffer.
1725          */
1726         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1727                                    DMA_TO_DEVICE);
1728         if (ret)
1729                 return ret;
1730
1731         /* Save the workarea address since it is updated in order to perform
1732          * the concatenation
1733          */
1734         save = src.address;
1735
1736         /* Copy the ECC modulus */
1737         ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1738                                 CCP_ECC_OPERAND_SIZE, true);
1739         src.address += CCP_ECC_OPERAND_SIZE;
1740
1741         /* Copy the first operand */
1742         ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1743                                 ecc->u.mm.operand_1_len,
1744                                 CCP_ECC_OPERAND_SIZE, true);
1745         src.address += CCP_ECC_OPERAND_SIZE;
1746
1747         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1748                 /* Copy the second operand */
1749                 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1750                                         ecc->u.mm.operand_2_len,
1751                                         CCP_ECC_OPERAND_SIZE, true);
1752                 src.address += CCP_ECC_OPERAND_SIZE;
1753         }
1754
1755         /* Restore the workarea address */
1756         src.address = save;
1757
1758         /* Prepare the output area for the operation */
1759         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1760                                    DMA_FROM_DEVICE);
1761         if (ret)
1762                 goto e_src;
1763
1764         op.soc = 1;
1765         op.src.u.dma.address = src.dma.address;
1766         op.src.u.dma.offset = 0;
1767         op.src.u.dma.length = src.length;
1768         op.dst.u.dma.address = dst.dma.address;
1769         op.dst.u.dma.offset = 0;
1770         op.dst.u.dma.length = dst.length;
1771
1772         op.u.ecc.function = cmd->u.ecc.function;
1773
1774         ret = ccp_perform_ecc(&op);
1775         if (ret) {
1776                 cmd->engine_error = cmd_q->cmd_error;
1777                 goto e_dst;
1778         }
1779
1780         ecc->ecc_result = le16_to_cpup(
1781                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1782         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1783                 ret = -EIO;
1784                 goto e_dst;
1785         }
1786
1787         /* Save the ECC result */
1788         ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1789
1790 e_dst:
1791         ccp_dm_free(&dst);
1792
1793 e_src:
1794         ccp_dm_free(&src);
1795
1796         return ret;
1797 }
1798
1799 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1800 {
1801         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1802         struct ccp_dm_workarea src, dst;
1803         struct ccp_op op;
1804         int ret;
1805         u8 *save;
1806
1807         if (!ecc->u.pm.point_1.x ||
1808             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1809             !ecc->u.pm.point_1.y ||
1810             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1811                 return -EINVAL;
1812
1813         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1814                 if (!ecc->u.pm.point_2.x ||
1815                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1816                     !ecc->u.pm.point_2.y ||
1817                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1818                         return -EINVAL;
1819         } else {
1820                 if (!ecc->u.pm.domain_a ||
1821                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1822                         return -EINVAL;
1823
1824                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1825                         if (!ecc->u.pm.scalar ||
1826                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1827                                 return -EINVAL;
1828         }
1829
1830         if (!ecc->u.pm.result.x ||
1831             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1832             !ecc->u.pm.result.y ||
1833             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1834                 return -EINVAL;
1835
1836         memset(&op, 0, sizeof(op));
1837         op.cmd_q = cmd_q;
1838         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1839
1840         /* Concatenate the modulus and the operands. Both the modulus and
1841          * the operands must be in little endian format.  Since the input
1842          * is in big endian format it must be converted and placed in a
1843          * fixed length buffer.
1844          */
1845         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1846                                    DMA_TO_DEVICE);
1847         if (ret)
1848                 return ret;
1849
1850         /* Save the workarea address since it is updated in order to perform
1851          * the concatenation
1852          */
1853         save = src.address;
1854
1855         /* Copy the ECC modulus */
1856         ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1857                                 CCP_ECC_OPERAND_SIZE, true);
1858         src.address += CCP_ECC_OPERAND_SIZE;
1859
1860         /* Copy the first point X and Y coordinate */
1861         ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1862                                 ecc->u.pm.point_1.x_len,
1863                                 CCP_ECC_OPERAND_SIZE, true);
1864         src.address += CCP_ECC_OPERAND_SIZE;
1865         ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1866                                 ecc->u.pm.point_1.y_len,
1867                                 CCP_ECC_OPERAND_SIZE, true);
1868         src.address += CCP_ECC_OPERAND_SIZE;
1869
1870         /* Set the first point Z coordianate to 1 */
1871         *(src.address) = 0x01;
1872         src.address += CCP_ECC_OPERAND_SIZE;
1873
1874         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1875                 /* Copy the second point X and Y coordinate */
1876                 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1877                                         ecc->u.pm.point_2.x_len,
1878                                         CCP_ECC_OPERAND_SIZE, true);
1879                 src.address += CCP_ECC_OPERAND_SIZE;
1880                 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1881                                         ecc->u.pm.point_2.y_len,
1882                                         CCP_ECC_OPERAND_SIZE, true);
1883                 src.address += CCP_ECC_OPERAND_SIZE;
1884
1885                 /* Set the second point Z coordianate to 1 */
1886                 *(src.address) = 0x01;
1887                 src.address += CCP_ECC_OPERAND_SIZE;
1888         } else {
1889                 /* Copy the Domain "a" parameter */
1890                 ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1891                                         ecc->u.pm.domain_a_len,
1892                                         CCP_ECC_OPERAND_SIZE, true);
1893                 src.address += CCP_ECC_OPERAND_SIZE;
1894
1895                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1896                         /* Copy the scalar value */
1897                         ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1898                                                 ecc->u.pm.scalar_len,
1899                                                 CCP_ECC_OPERAND_SIZE, true);
1900                         src.address += CCP_ECC_OPERAND_SIZE;
1901                 }
1902         }
1903
1904         /* Restore the workarea address */
1905         src.address = save;
1906
1907         /* Prepare the output area for the operation */
1908         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1909                                    DMA_FROM_DEVICE);
1910         if (ret)
1911                 goto e_src;
1912
1913         op.soc = 1;
1914         op.src.u.dma.address = src.dma.address;
1915         op.src.u.dma.offset = 0;
1916         op.src.u.dma.length = src.length;
1917         op.dst.u.dma.address = dst.dma.address;
1918         op.dst.u.dma.offset = 0;
1919         op.dst.u.dma.length = dst.length;
1920
1921         op.u.ecc.function = cmd->u.ecc.function;
1922
1923         ret = ccp_perform_ecc(&op);
1924         if (ret) {
1925                 cmd->engine_error = cmd_q->cmd_error;
1926                 goto e_dst;
1927         }
1928
1929         ecc->ecc_result = le16_to_cpup(
1930                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1931         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1932                 ret = -EIO;
1933                 goto e_dst;
1934         }
1935
1936         /* Save the workarea address since it is updated as we walk through
1937          * to copy the point math result
1938          */
1939         save = dst.address;
1940
1941         /* Save the ECC result X and Y coordinates */
1942         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1943                                 CCP_ECC_MODULUS_BYTES);
1944         dst.address += CCP_ECC_OUTPUT_SIZE;
1945         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1946                                 CCP_ECC_MODULUS_BYTES);
1947         dst.address += CCP_ECC_OUTPUT_SIZE;
1948
1949         /* Restore the workarea address */
1950         dst.address = save;
1951
1952 e_dst:
1953         ccp_dm_free(&dst);
1954
1955 e_src:
1956         ccp_dm_free(&src);
1957
1958         return ret;
1959 }
1960
1961 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1962 {
1963         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1964
1965         ecc->ecc_result = 0;
1966
1967         if (!ecc->mod ||
1968             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1969                 return -EINVAL;
1970
1971         switch (ecc->function) {
1972         case CCP_ECC_FUNCTION_MMUL_384BIT:
1973         case CCP_ECC_FUNCTION_MADD_384BIT:
1974         case CCP_ECC_FUNCTION_MINV_384BIT:
1975                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1976
1977         case CCP_ECC_FUNCTION_PADD_384BIT:
1978         case CCP_ECC_FUNCTION_PMUL_384BIT:
1979         case CCP_ECC_FUNCTION_PDBL_384BIT:
1980                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1981
1982         default:
1983                 return -EINVAL;
1984         }
1985 }
1986
1987 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1988 {
1989         int ret;
1990
1991         cmd->engine_error = 0;
1992         cmd_q->cmd_error = 0;
1993         cmd_q->int_rcvd = 0;
1994         cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
1995
1996         switch (cmd->engine) {
1997         case CCP_ENGINE_AES:
1998                 ret = ccp_run_aes_cmd(cmd_q, cmd);
1999                 break;
2000         case CCP_ENGINE_XTS_AES_128:
2001                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2002                 break;
2003         case CCP_ENGINE_SHA:
2004                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2005                 break;
2006         case CCP_ENGINE_RSA:
2007                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2008                 break;
2009         case CCP_ENGINE_PASSTHRU:
2010                 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2011                 break;
2012         case CCP_ENGINE_ECC:
2013                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2014                 break;
2015         default:
2016                 ret = -EINVAL;
2017         }
2018
2019         return ret;
2020 }