drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/pci_ids.h>
  17 #include <linux/kthread.h>
  18 #include <linux/sched.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/mutex.h>
  22 #include <linux/delay.h>
  23 #include <linux/ccp.h>
  24 #include <linux/scatterlist.h>
  25 #include <crypto/scatterwalk.h>
  26
  27 #include "ccp-dev.h"
  28
  29
  30 enum ccp_memtype {
  31         CCP_MEMTYPE_SYSTEM = 0,
  32         CCP_MEMTYPE_KSB,
  33         CCP_MEMTYPE_LOCAL,
  34         CCP_MEMTYPE__LAST,
  35 };
  36
  37 struct ccp_dma_info {
  38         dma_addr_t address;
  39         unsigned int offset;
  40         unsigned int length;
  41         enum dma_data_direction dir;
  42 };
  43
  44 struct ccp_dm_workarea {
  45         struct device *dev;
  46         struct dma_pool *dma_pool;
  47         unsigned int length;
  48
  49         u8 *address;
  50         struct ccp_dma_info dma;
  51 };
  52
  53 struct ccp_sg_workarea {
  54         struct scatterlist *sg;
  55         unsigned int nents;
  56         unsigned int length;
  57
  58         struct scatterlist *dma_sg;
  59         struct device *dma_dev;
  60         unsigned int dma_count;
  61         enum dma_data_direction dma_dir;
  62
  63         unsigned int sg_used;
  64
  65         u64 bytes_left;
  66 };
  67
  68 struct ccp_data {
  69         struct ccp_sg_workarea sg_wa;
  70         struct ccp_dm_workarea dm_wa;
  71 };
  72
  73 struct ccp_mem {
  74         enum ccp_memtype type;
  75         union {
  76                 struct ccp_dma_info dma;
  77                 u32 ksb;
  78         } u;
  79 };
  80
  81 struct ccp_aes_op {
  82         enum ccp_aes_type type;
  83         enum ccp_aes_mode mode;
  84         enum ccp_aes_action action;
  85 };
  86
  87 struct ccp_xts_aes_op {
  88         enum ccp_aes_action action;
  89         enum ccp_xts_aes_unit_size unit_size;
  90 };
  91
  92 struct ccp_sha_op {
  93         enum ccp_sha_type type;
  94         u64 msg_bits;
  95 };
  96
  97 struct ccp_rsa_op {
  98         u32 mod_size;
  99         u32 input_len;
 100 };
 101
 102 struct ccp_passthru_op {
 103         enum ccp_passthru_bitwise bit_mod;
 104         enum ccp_passthru_byteswap byte_swap;
 105 };
 106
 107 struct ccp_ecc_op {
 108         enum ccp_ecc_function function;
 109 };
 110
 111 struct ccp_op {
 112         struct ccp_cmd_queue *cmd_q;
 113
 114         u32 jobid;
 115         u32 ioc;
 116         u32 soc;
 117         u32 ksb_key;
 118         u32 ksb_ctx;
 119         u32 init;
 120         u32 eom;
 121
 122         struct ccp_mem src;
 123         struct ccp_mem dst;
 124
 125         union {
 126                 struct ccp_aes_op aes;
 127                 struct ccp_xts_aes_op xts;
 128                 struct ccp_sha_op sha;
 129                 struct ccp_rsa_op rsa;
 130                 struct ccp_passthru_op passthru;
 131                 struct ccp_ecc_op ecc;
 132         } u;
 133 };
 134
 135 /* The CCP cannot perform zero-length sha operations so the caller
 136  * is required to buffer data for the final operation.  However, a
 137  * sha operation for a message with a total length of zero is valid
 138  * so known values are required to supply the result.
 139  */
 140 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = {
 141         0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
 142         0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
 143         0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00,
 144         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 145 };
 146
 147 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = {
 148         0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9,
 149         0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4,
 150         0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a,
 151         0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00,
 152 };
 153
 154 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = {
 155         0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
 156         0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
 157         0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
 158         0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55,
 159 };
 160
 161 static u32 ccp_addr_lo(struct ccp_dma_info *info)
 162 {
 163         return lower_32_bits(info->address + info->offset);
 164 }
 165
 166 static u32 ccp_addr_hi(struct ccp_dma_info *info)
 167 {
 168         return upper_32_bits(info->address + info->offset) & 0x0000ffff;
 169 }
 170
 171 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
 172 {
 173         struct ccp_cmd_queue *cmd_q = op->cmd_q;
 174         struct ccp_device *ccp = cmd_q->ccp;
 175         void __iomem *cr_addr;
 176         u32 cr0, cmd;
 177         unsigned int i;
 178         int ret = 0;
 179
 180         /* We could read a status register to see how many free slots
 181          * are actually available, but reading that register resets it
 182          * and you could lose some error information.
 183          */
 184         cmd_q->free_slots--;
 185
 186         cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
 187               | (op->jobid << REQ0_JOBID_SHIFT)
 188               | REQ0_WAIT_FOR_WRITE;
 189
 190         if (op->soc)
 191                 cr0 |= REQ0_STOP_ON_COMPLETE
 192                        | REQ0_INT_ON_COMPLETE;
 193
 194         if (op->ioc || !cmd_q->free_slots)
 195                 cr0 |= REQ0_INT_ON_COMPLETE;
 196
 197         /* Start at CMD_REQ1 */
 198         cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
 199
 200         mutex_lock(&ccp->req_mutex);
 201
 202         /* Write CMD_REQ1 through CMD_REQx first */
 203         for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
 204                 iowrite32(*(cr + i), cr_addr);
 205
 206         /* Tell the CCP to start */
 207         wmb();
 208         iowrite32(cr0, ccp->io_regs + CMD_REQ0);
 209
 210         mutex_unlock(&ccp->req_mutex);
 211
 212         if (cr0 & REQ0_INT_ON_COMPLETE) {
 213                 /* Wait for the job to complete */
 214                 ret = wait_event_interruptible(cmd_q->int_queue,
 215                                                cmd_q->int_rcvd);
 216                 if (ret || cmd_q->cmd_error) {
 217                         /* On error delete all related jobs from the queue */
 218                         cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
 219                               | op->jobid;
 220
 221                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 222
 223                         if (!ret)
 224                                 ret = -EIO;
 225                 } else if (op->soc) {
 226                         /* Delete just head job from the queue on SoC */
 227                         cmd = DEL_Q_ACTIVE
 228                               | (cmd_q->id << DEL_Q_ID_SHIFT)
 229                               | op->jobid;
 230
 231                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 232                 }
 233
 234                 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
 235
 236                 cmd_q->int_rcvd = 0;
 237         }
 238
 239         return ret;
 240 }
 241
 242 static int ccp_perform_aes(struct ccp_op *op)
 243 {
 244         u32 cr[6];
 245
 246         /* Fill out the register contents for REQ1 through REQ6 */
 247         cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
 248                 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
 249                 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
 250                 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
 251                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
 252         cr[1] = op->src.u.dma.length - 1;
 253         cr[2] = ccp_addr_lo(&op->src.u.dma);
 254         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 255                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 256                 | ccp_addr_hi(&op->src.u.dma);
 257         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 258         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 259                 | ccp_addr_hi(&op->dst.u.dma);
 260
 261         if (op->u.aes.mode == CCP_AES_MODE_CFB)
 262                 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
 263
 264         if (op->eom)
 265                 cr[0] |= REQ1_EOM;
 266
 267         if (op->init)
 268                 cr[0] |= REQ1_INIT;
 269
 270         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 271 }
 272
 273 static int ccp_perform_xts_aes(struct ccp_op *op)
 274 {
 275         u32 cr[6];
 276
 277         /* Fill out the register contents for REQ1 through REQ6 */
 278         cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
 279                 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
 280                 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
 281                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
 282         cr[1] = op->src.u.dma.length - 1;
 283         cr[2] = ccp_addr_lo(&op->src.u.dma);
 284         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 285                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 286                 | ccp_addr_hi(&op->src.u.dma);
 287         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 288         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 289                 | ccp_addr_hi(&op->dst.u.dma);
 290
 291         if (op->eom)
 292                 cr[0] |= REQ1_EOM;
 293
 294         if (op->init)
 295                 cr[0] |= REQ1_INIT;
 296
 297         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 298 }
 299
 300 static int ccp_perform_sha(struct ccp_op *op)
 301 {
 302         u32 cr[6];
 303
 304         /* Fill out the register contents for REQ1 through REQ6 */
 305         cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
 306                 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
 307                 | REQ1_INIT;
 308         cr[1] = op->src.u.dma.length - 1;
 309         cr[2] = ccp_addr_lo(&op->src.u.dma);
 310         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 311                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 312                 | ccp_addr_hi(&op->src.u.dma);
 313
 314         if (op->eom) {
 315                 cr[0] |= REQ1_EOM;
 316                 cr[4] = lower_32_bits(op->u.sha.msg_bits);
 317                 cr[5] = upper_32_bits(op->u.sha.msg_bits);
 318         } else {
 319                 cr[4] = 0;
 320                 cr[5] = 0;
 321         }
 322
 323         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 324 }
 325
 326 static int ccp_perform_rsa(struct ccp_op *op)
 327 {
 328         u32 cr[6];
 329
 330         /* Fill out the register contents for REQ1 through REQ6 */
 331         cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
 332                 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
 333                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT)
 334                 | REQ1_EOM;
 335         cr[1] = op->u.rsa.input_len - 1;
 336         cr[2] = ccp_addr_lo(&op->src.u.dma);
 337         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 338                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 339                 | ccp_addr_hi(&op->src.u.dma);
 340         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 341         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 342                 | ccp_addr_hi(&op->dst.u.dma);
 343
 344         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 345 }
 346
 347 static int ccp_perform_passthru(struct ccp_op *op)
 348 {
 349         u32 cr[6];
 350
 351         /* Fill out the register contents for REQ1 through REQ6 */
 352         cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
 353                 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
 354                 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
 355
 356         if (op->src.type == CCP_MEMTYPE_SYSTEM)
 357                 cr[1] = op->src.u.dma.length - 1;
 358         else
 359                 cr[1] = op->dst.u.dma.length - 1;
 360
 361         if (op->src.type == CCP_MEMTYPE_SYSTEM) {
 362                 cr[2] = ccp_addr_lo(&op->src.u.dma);
 363                 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 364                         | ccp_addr_hi(&op->src.u.dma);
 365
 366                 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
 367                         cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
 368         } else {
 369                 cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
 370                 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
 371         }
 372
 373         if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
 374                 cr[4] = ccp_addr_lo(&op->dst.u.dma);
 375                 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 376                         | ccp_addr_hi(&op->dst.u.dma);
 377         } else {
 378                 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
 379                 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
 380         }
 381
 382         if (op->eom)
 383                 cr[0] |= REQ1_EOM;
 384
 385         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 386 }
 387
 388 static int ccp_perform_ecc(struct ccp_op *op)
 389 {
 390         u32 cr[6];
 391
 392         /* Fill out the register contents for REQ1 through REQ6 */
 393         cr[0] = REQ1_ECC_AFFINE_CONVERT
 394                 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
 395                 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
 396                 | REQ1_EOM;
 397         cr[1] = op->src.u.dma.length - 1;
 398         cr[2] = ccp_addr_lo(&op->src.u.dma);
 399         cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 400                 | ccp_addr_hi(&op->src.u.dma);
 401         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 402         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 403                 | ccp_addr_hi(&op->dst.u.dma);
 404
 405         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 406 }
 407
 408 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
 409 {
 410         int start;
 411
 412         for (;;) {
 413                 mutex_lock(&ccp->ksb_mutex);
 414
 415                 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
 416                                                         ccp->ksb_count,
 417                                                         ccp->ksb_start,
 418                                                         count, 0);
 419                 if (start <= ccp->ksb_count) {
 420                         bitmap_set(ccp->ksb, start, count);
 421
 422                         mutex_unlock(&ccp->ksb_mutex);
 423                         break;
 424                 }
 425
 426                 ccp->ksb_avail = 0;
 427
 428                 mutex_unlock(&ccp->ksb_mutex);
 429
 430                 /* Wait for KSB entries to become available */
 431                 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
 432                         return 0;
 433         }
 434
 435         return KSB_START + start;
 436 }
 437
 438 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
 439                          unsigned int count)
 440 {
 441         if (!start)
 442                 return;
 443
 444         mutex_lock(&ccp->ksb_mutex);
 445
 446         bitmap_clear(ccp->ksb, start - KSB_START, count);
 447
 448         ccp->ksb_avail = 1;
 449
 450         mutex_unlock(&ccp->ksb_mutex);
 451
 452         wake_up_interruptible_all(&ccp->ksb_queue);
 453 }
 454
 455 static u32 ccp_gen_jobid(struct ccp_device *ccp)
 456 {
 457         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
 458 }
 459
 460 static void ccp_sg_free(struct ccp_sg_workarea *wa)
 461 {
 462         if (wa->dma_count)
 463                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
 464
 465         wa->dma_count = 0;
 466 }
 467
 468 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
 469                                 struct scatterlist *sg, u64 len,
 470                                 enum dma_data_direction dma_dir)
 471 {
 472         memset(wa, 0, sizeof(*wa));
 473
 474         wa->sg = sg;
 475         if (!sg)
 476                 return 0;
 477
 478         wa->nents = sg_nents(sg);
 479         wa->length = sg->length;
 480         wa->bytes_left = len;
 481         wa->sg_used = 0;
 482
 483         if (len == 0)
 484                 return 0;
 485
 486         if (dma_dir == DMA_NONE)
 487                 return 0;
 488
 489         wa->dma_sg = sg;
 490         wa->dma_dev = dev;
 491         wa->dma_dir = dma_dir;
 492         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 493         if (!wa->dma_count)
 494                 return -ENOMEM;
 495
 496
 497         return 0;
 498 }
 499
 500 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 501 {
 502         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 503
 504         if (!wa->sg)
 505                 return;
 506
 507         wa->sg_used += nbytes;
 508         wa->bytes_left -= nbytes;
 509         if (wa->sg_used == wa->sg->length) {
 510                 wa->sg = sg_next(wa->sg);
 511                 wa->sg_used = 0;
 512         }
 513 }
 514
 515 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 516 {
 517         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 518                 if (wa->address)
 519                         dma_pool_free(wa->dma_pool, wa->address,
 520                                       wa->dma.address);
 521         } else {
 522                 if (wa->dma.address)
 523                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 524                                          wa->dma.dir);
 525                 kfree(wa->address);
 526         }
 527
 528         wa->address = NULL;
 529         wa->dma.address = 0;
 530 }
 531
 532 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 533                                 struct ccp_cmd_queue *cmd_q,
 534                                 unsigned int len,
 535                                 enum dma_data_direction dir)
 536 {
 537         memset(wa, 0, sizeof(*wa));
 538
 539         if (!len)
 540                 return 0;
 541
 542         wa->dev = cmd_q->ccp->dev;
 543         wa->length = len;
 544
 545         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 546                 wa->dma_pool = cmd_q->dma_pool;
 547
 548                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 549                                              &wa->dma.address);
 550                 if (!wa->address)
 551                         return -ENOMEM;
 552
 553                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 554
 555                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 556         } else {
 557                 wa->address = kzalloc(len, GFP_KERNEL);
 558                 if (!wa->address)
 559                         return -ENOMEM;
 560
 561                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 562                                                  dir);
 563                 if (!wa->dma.address)
 564                         return -ENOMEM;
 565
 566                 wa->dma.length = len;
 567         }
 568         wa->dma.dir = dir;
 569
 570         return 0;
 571 }
 572
 573 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 574                             struct scatterlist *sg, unsigned int sg_offset,
 575                             unsigned int len)
 576 {
 577         WARN_ON(!wa->address);
 578
 579         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 580                                  0);
 581 }
 582
 583 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 584                             struct scatterlist *sg, unsigned int sg_offset,
 585                             unsigned int len)
 586 {
 587         WARN_ON(!wa->address);
 588
 589         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 590                                  1);
 591 }
 592
 593 static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 594                                     struct scatterlist *sg,
 595                                     unsigned int len, unsigned int se_len,
 596                                     bool sign_extend)
 597 {
 598         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 599         u8 buffer[CCP_REVERSE_BUF_SIZE];
 600
 601         BUG_ON(se_len > sizeof(buffer));
 602
 603         sg_offset = len;
 604         dm_offset = 0;
 605         nbytes = len;
 606         while (nbytes) {
 607                 ksb_len = min_t(unsigned int, nbytes, se_len);
 608                 sg_offset -= ksb_len;
 609
 610                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
 611                 for (i = 0; i < ksb_len; i++)
 612                         wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
 613
 614                 dm_offset += ksb_len;
 615                 nbytes -= ksb_len;
 616
 617                 if ((ksb_len != se_len) && sign_extend) {
 618                         /* Must sign-extend to nearest sign-extend length */
 619                         if (wa->address[dm_offset - 1] & 0x80)
 620                                 memset(wa->address + dm_offset, 0xff,
 621                                        se_len - ksb_len);
 622                 }
 623         }
 624 }
 625
 626 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 627                                     struct scatterlist *sg,
 628                                     unsigned int len)
 629 {
 630         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 631         u8 buffer[CCP_REVERSE_BUF_SIZE];
 632
 633         sg_offset = 0;
 634         dm_offset = len;
 635         nbytes = len;
 636         while (nbytes) {
 637                 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
 638                 dm_offset -= ksb_len;
 639
 640                 for (i = 0; i < ksb_len; i++)
 641                         buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
 642                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
 643
 644                 sg_offset += ksb_len;
 645                 nbytes -= ksb_len;
 646         }
 647 }
 648
 649 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 650 {
 651         ccp_dm_free(&data->dm_wa);
 652         ccp_sg_free(&data->sg_wa);
 653 }
 654
 655 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 656                          struct scatterlist *sg, u64 sg_len,
 657                          unsigned int dm_len,
 658                          enum dma_data_direction dir)
 659 {
 660         int ret;
 661
 662         memset(data, 0, sizeof(*data));
 663
 664         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 665                                    dir);
 666         if (ret)
 667                 goto e_err;
 668
 669         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 670         if (ret)
 671                 goto e_err;
 672
 673         return 0;
 674
 675 e_err:
 676         ccp_free_data(data, cmd_q);
 677
 678         return ret;
 679 }
 680
 681 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 682 {
 683         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 684         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 685         unsigned int buf_count, nbytes;
 686
 687         /* Clear the buffer if setting it */
 688         if (!from)
 689                 memset(dm_wa->address, 0, dm_wa->length);
 690
 691         if (!sg_wa->sg)
 692                 return 0;
 693
 694         /* Perform the copy operation
 695          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 696          *   an unsigned int
 697          */
 698         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 699         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 700                                  nbytes, from);
 701
 702         /* Update the structures and generate the count */
 703         buf_count = 0;
 704         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 705                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 706                              dm_wa->length - buf_count);
 707                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 708
 709                 buf_count += nbytes;
 710                 ccp_update_sg_workarea(sg_wa, nbytes);
 711         }
 712
 713         return buf_count;
 714 }
 715
 716 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 717 {
 718         return ccp_queue_buf(data, 0);
 719 }
 720
 721 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 722 {
 723         return ccp_queue_buf(data, 1);
 724 }
 725
 726 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 727                              struct ccp_op *op, unsigned int block_size,
 728                              bool blocksize_op)
 729 {
 730         unsigned int sg_src_len, sg_dst_len, op_len;
 731
 732         /* The CCP can only DMA from/to one address each per operation. This
 733          * requires that we find the smallest DMA area between the source
 734          * and destination. The resulting len values will always be <= UINT_MAX
 735          * because the dma length is an unsigned int.
 736          */
 737         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 738         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 739
 740         if (dst) {
 741                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 742                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 743                 op_len = min(sg_src_len, sg_dst_len);
 744         } else
 745                 op_len = sg_src_len;
 746
 747         /* The data operation length will be at least block_size in length
 748          * or the smaller of available sg room remaining for the source or
 749          * the destination
 750          */
 751         op_len = max(op_len, block_size);
 752
 753         /* Unless we have to buffer data, there's no reason to wait */
 754         op->soc = 0;
 755
 756         if (sg_src_len < block_size) {
 757                 /* Not enough data in the sg element, so it
 758                  * needs to be buffered into a blocksize chunk
 759                  */
 760                 int cp_len = ccp_fill_queue_buf(src);
 761
 762                 op->soc = 1;
 763                 op->src.u.dma.address = src->dm_wa.dma.address;
 764                 op->src.u.dma.offset = 0;
 765                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 766         } else {
 767                 /* Enough data in the sg element, but we need to
 768                  * adjust for any previously copied data
 769                  */
 770                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 771                 op->src.u.dma.offset = src->sg_wa.sg_used;
 772                 op->src.u.dma.length = op_len & ~(block_size - 1);
 773
 774                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 775         }
 776
 777         if (dst) {
 778                 if (sg_dst_len < block_size) {
 779                         /* Not enough room in the sg element or we're on the
 780                          * last piece of data (when using padding), so the
 781                          * output needs to be buffered into a blocksize chunk
 782                          */
 783                         op->soc = 1;
 784                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 785                         op->dst.u.dma.offset = 0;
 786                         op->dst.u.dma.length = op->src.u.dma.length;
 787                 } else {
 788                         /* Enough room in the sg element, but we need to
 789                          * adjust for any previously used area
 790                          */
 791                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 792                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 793                         op->dst.u.dma.length = op->src.u.dma.length;
 794                 }
 795         }
 796 }
 797
 798 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 799                              struct ccp_op *op)
 800 {
 801         op->init = 0;
 802
 803         if (dst) {
 804                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 805                         ccp_empty_queue_buf(dst);
 806                 else
 807                         ccp_update_sg_workarea(&dst->sg_wa,
 808                                                op->dst.u.dma.length);
 809         }
 810 }
 811
 812 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
 813                                 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 814                                 u32 byte_swap, bool from)
 815 {
 816         struct ccp_op op;
 817
 818         memset(&op, 0, sizeof(op));
 819
 820         op.cmd_q = cmd_q;
 821         op.jobid = jobid;
 822         op.eom = 1;
 823
 824         if (from) {
 825                 op.soc = 1;
 826                 op.src.type = CCP_MEMTYPE_KSB;
 827                 op.src.u.ksb = ksb;
 828                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 829                 op.dst.u.dma.address = wa->dma.address;
 830                 op.dst.u.dma.length = wa->length;
 831         } else {
 832                 op.src.type = CCP_MEMTYPE_SYSTEM;
 833                 op.src.u.dma.address = wa->dma.address;
 834                 op.src.u.dma.length = wa->length;
 835                 op.dst.type = CCP_MEMTYPE_KSB;
 836                 op.dst.u.ksb = ksb;
 837         }
 838
 839         op.u.passthru.byte_swap = byte_swap;
 840
 841         return ccp_perform_passthru(&op);
 842 }
 843
 844 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
 845                            struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 846                            u32 byte_swap)
 847 {
 848         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
 849 }
 850
 851 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
 852                              struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 853                              u32 byte_swap)
 854 {
 855         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
 856 }
 857
 858 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 859                                 struct ccp_cmd *cmd)
 860 {
 861         struct ccp_aes_engine *aes = &cmd->u.aes;
 862         struct ccp_dm_workarea key, ctx;
 863         struct ccp_data src;
 864         struct ccp_op op;
 865         unsigned int dm_offset;
 866         int ret;
 867
 868         if (!((aes->key_len == AES_KEYSIZE_128) ||
 869               (aes->key_len == AES_KEYSIZE_192) ||
 870               (aes->key_len == AES_KEYSIZE_256)))
 871                 return -EINVAL;
 872
 873         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 874                 return -EINVAL;
 875
 876         if (aes->iv_len != AES_BLOCK_SIZE)
 877                 return -EINVAL;
 878
 879         if (!aes->key || !aes->iv || !aes->src)
 880                 return -EINVAL;
 881
 882         if (aes->cmac_final) {
 883                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 884                         return -EINVAL;
 885
 886                 if (!aes->cmac_key)
 887                         return -EINVAL;
 888         }
 889
 890         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
 891         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
 892
 893         ret = -EIO;
 894         memset(&op, 0, sizeof(op));
 895         op.cmd_q = cmd_q;
 896         op.jobid = ccp_gen_jobid(cmd_q->ccp);
 897         op.ksb_key = cmd_q->ksb_key;
 898         op.ksb_ctx = cmd_q->ksb_ctx;
 899         op.init = 1;
 900         op.u.aes.type = aes->type;
 901         op.u.aes.mode = aes->mode;
 902         op.u.aes.action = aes->action;
 903
 904         /* All supported key sizes fit in a single (32-byte) KSB entry
 905          * and must be in little endian format. Use the 256-bit byte
 906          * swap passthru option to convert from big endian to little
 907          * endian.
 908          */
 909         ret = ccp_init_dm_workarea(&key, cmd_q,
 910                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
 911                                    DMA_TO_DEVICE);
 912         if (ret)
 913                 return ret;
 914
 915         dm_offset = CCP_KSB_BYTES - aes->key_len;
 916         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 917         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
 918                               CCP_PASSTHRU_BYTESWAP_256BIT);
 919         if (ret) {
 920                 cmd->engine_error = cmd_q->cmd_error;
 921                 goto e_key;
 922         }
 923
 924         /* The AES context fits in a single (32-byte) KSB entry and
 925          * must be in little endian format. Use the 256-bit byte swap
 926          * passthru option to convert from big endian to little endian.
 927          */
 928         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 929                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
 930                                    DMA_BIDIRECTIONAL);
 931         if (ret)
 932                 goto e_key;
 933
 934         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 935         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 936         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 937                               CCP_PASSTHRU_BYTESWAP_256BIT);
 938         if (ret) {
 939                 cmd->engine_error = cmd_q->cmd_error;
 940                 goto e_ctx;
 941         }
 942
 943         /* Send data to the CCP AES engine */
 944         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 945                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 946         if (ret)
 947                 goto e_ctx;
 948
 949         while (src.sg_wa.bytes_left) {
 950                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 951                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 952                         op.eom = 1;
 953
 954                         /* Push the K1/K2 key to the CCP now */
 955                         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
 956                                                 op.ksb_ctx,
 957                                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 958                         if (ret) {
 959                                 cmd->engine_error = cmd_q->cmd_error;
 960                                 goto e_src;
 961                         }
 962
 963                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 964                                         aes->cmac_key_len);
 965                         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 966                                               CCP_PASSTHRU_BYTESWAP_256BIT);
 967                         if (ret) {
 968                                 cmd->engine_error = cmd_q->cmd_error;
 969                                 goto e_src;
 970                         }
 971                 }
 972
 973                 ret = ccp_perform_aes(&op);
 974                 if (ret) {
 975                         cmd->engine_error = cmd_q->cmd_error;
 976                         goto e_src;
 977                 }
 978
 979                 ccp_process_data(&src, NULL, &op);
 980         }
 981
 982         /* Retrieve the AES context - convert from LE to BE using
 983          * 32-byte (256-bit) byteswapping
 984          */
 985         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 986                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 987         if (ret) {
 988                 cmd->engine_error = cmd_q->cmd_error;
 989                 goto e_src;
 990         }
 991
 992         /* ...but we only need AES_BLOCK_SIZE bytes */
 993         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 994         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 995
 996 e_src:
 997         ccp_free_data(&src, cmd_q);
 998
 999 e_ctx:
1000         ccp_dm_free(&ctx);
1001
1002 e_key:
1003         ccp_dm_free(&key);
1004
1005         return ret;
1006 }
1007
1008 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1009 {
1010         struct ccp_aes_engine *aes = &cmd->u.aes;
1011         struct ccp_dm_workarea key, ctx;
1012         struct ccp_data src, dst;
1013         struct ccp_op op;
1014         unsigned int dm_offset;
1015         bool in_place = false;
1016         int ret;
1017
1018         if (aes->mode == CCP_AES_MODE_CMAC)
1019                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
1020
1021         if (!((aes->key_len == AES_KEYSIZE_128) ||
1022               (aes->key_len == AES_KEYSIZE_192) ||
1023               (aes->key_len == AES_KEYSIZE_256)))
1024                 return -EINVAL;
1025
1026         if (((aes->mode == CCP_AES_MODE_ECB) ||
1027              (aes->mode == CCP_AES_MODE_CBC) ||
1028              (aes->mode == CCP_AES_MODE_CFB)) &&
1029             (aes->src_len & (AES_BLOCK_SIZE - 1)))
1030                 return -EINVAL;
1031
1032         if (!aes->key || !aes->src || !aes->dst)
1033                 return -EINVAL;
1034
1035         if (aes->mode != CCP_AES_MODE_ECB) {
1036                 if (aes->iv_len != AES_BLOCK_SIZE)
1037                         return -EINVAL;
1038
1039                 if (!aes->iv)
1040                         return -EINVAL;
1041         }
1042
1043         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
1044         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
1045
1046         ret = -EIO;
1047         memset(&op, 0, sizeof(op));
1048         op.cmd_q = cmd_q;
1049         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1050         op.ksb_key = cmd_q->ksb_key;
1051         op.ksb_ctx = cmd_q->ksb_ctx;
1052         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
1053         op.u.aes.type = aes->type;
1054         op.u.aes.mode = aes->mode;
1055         op.u.aes.action = aes->action;
1056
1057         /* All supported key sizes fit in a single (32-byte) KSB entry
1058          * and must be in little endian format. Use the 256-bit byte
1059          * swap passthru option to convert from big endian to little
1060          * endian.
1061          */
1062         ret = ccp_init_dm_workarea(&key, cmd_q,
1063                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1064                                    DMA_TO_DEVICE);
1065         if (ret)
1066                 return ret;
1067
1068         dm_offset = CCP_KSB_BYTES - aes->key_len;
1069         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
1070         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1071                               CCP_PASSTHRU_BYTESWAP_256BIT);
1072         if (ret) {
1073                 cmd->engine_error = cmd_q->cmd_error;
1074                 goto e_key;
1075         }
1076
1077         /* The AES context fits in a single (32-byte) KSB entry and
1078          * must be in little endian format. Use the 256-bit byte swap
1079          * passthru option to convert from big endian to little endian.
1080          */
1081         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1082                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1083                                    DMA_BIDIRECTIONAL);
1084         if (ret)
1085                 goto e_key;
1086
1087         if (aes->mode != CCP_AES_MODE_ECB) {
1088                 /* Load the AES context - conver to LE */
1089                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1090                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1091                 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1092                                       CCP_PASSTHRU_BYTESWAP_256BIT);
1093                 if (ret) {
1094                         cmd->engine_error = cmd_q->cmd_error;
1095                         goto e_ctx;
1096                 }
1097         }
1098
1099         /* Prepare the input and output data workareas. For in-place
1100          * operations we need to set the dma direction to BIDIRECTIONAL
1101          * and copy the src workarea to the dst workarea.
1102          */
1103         if (sg_virt(aes->src) == sg_virt(aes->dst))
1104                 in_place = true;
1105
1106         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1107                             AES_BLOCK_SIZE,
1108                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1109         if (ret)
1110                 goto e_ctx;
1111
1112         if (in_place)
1113                 dst = src;
1114         else {
1115                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1116                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1117                 if (ret)
1118                         goto e_src;
1119         }
1120
1121         /* Send data to the CCP AES engine */
1122         while (src.sg_wa.bytes_left) {
1123                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1124                 if (!src.sg_wa.bytes_left) {
1125                         op.eom = 1;
1126
1127                         /* Since we don't retrieve the AES context in ECB
1128                          * mode we have to wait for the operation to complete
1129                          * on the last piece of data
1130                          */
1131                         if (aes->mode == CCP_AES_MODE_ECB)
1132                                 op.soc = 1;
1133                 }
1134
1135                 ret = ccp_perform_aes(&op);
1136                 if (ret) {
1137                         cmd->engine_error = cmd_q->cmd_error;
1138                         goto e_dst;
1139                 }
1140
1141                 ccp_process_data(&src, &dst, &op);
1142         }
1143
1144         if (aes->mode != CCP_AES_MODE_ECB) {
1145                 /* Retrieve the AES context - convert from LE to BE using
1146                  * 32-byte (256-bit) byteswapping
1147                  */
1148                 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1149                                         CCP_PASSTHRU_BYTESWAP_256BIT);
1150                 if (ret) {
1151                         cmd->engine_error = cmd_q->cmd_error;
1152                         goto e_dst;
1153                 }
1154
1155                 /* ...but we only need AES_BLOCK_SIZE bytes */
1156                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1157                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1158         }
1159
1160 e_dst:
1161         if (!in_place)
1162                 ccp_free_data(&dst, cmd_q);
1163
1164 e_src:
1165         ccp_free_data(&src, cmd_q);
1166
1167 e_ctx:
1168         ccp_dm_free(&ctx);
1169
1170 e_key:
1171         ccp_dm_free(&key);
1172
1173         return ret;
1174 }
1175
1176 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1177                                struct ccp_cmd *cmd)
1178 {
1179         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1180         struct ccp_dm_workarea key, ctx;
1181         struct ccp_data src, dst;
1182         struct ccp_op op;
1183         unsigned int unit_size, dm_offset;
1184         bool in_place = false;
1185         int ret;
1186
1187         switch (xts->unit_size) {
1188         case CCP_XTS_AES_UNIT_SIZE_16:
1189                 unit_size = 16;
1190                 break;
1191         case CCP_XTS_AES_UNIT_SIZE_512:
1192                 unit_size = 512;
1193                 break;
1194         case CCP_XTS_AES_UNIT_SIZE_1024:
1195                 unit_size = 1024;
1196                 break;
1197         case CCP_XTS_AES_UNIT_SIZE_2048:
1198                 unit_size = 2048;
1199                 break;
1200         case CCP_XTS_AES_UNIT_SIZE_4096:
1201                 unit_size = 4096;
1202                 break;
1203
1204         default:
1205                 return -EINVAL;
1206         }
1207
1208         if (xts->key_len != AES_KEYSIZE_128)
1209                 return -EINVAL;
1210
1211         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1212                 return -EINVAL;
1213
1214         if (xts->iv_len != AES_BLOCK_SIZE)
1215                 return -EINVAL;
1216
1217         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1218                 return -EINVAL;
1219
1220         BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
1221         BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
1222
1223         ret = -EIO;
1224         memset(&op, 0, sizeof(op));
1225         op.cmd_q = cmd_q;
1226         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1227         op.ksb_key = cmd_q->ksb_key;
1228         op.ksb_ctx = cmd_q->ksb_ctx;
1229         op.init = 1;
1230         op.u.xts.action = xts->action;
1231         op.u.xts.unit_size = xts->unit_size;
1232
1233         /* All supported key sizes fit in a single (32-byte) KSB entry
1234          * and must be in little endian format. Use the 256-bit byte
1235          * swap passthru option to convert from big endian to little
1236          * endian.
1237          */
1238         ret = ccp_init_dm_workarea(&key, cmd_q,
1239                                    CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1240                                    DMA_TO_DEVICE);
1241         if (ret)
1242                 return ret;
1243
1244         dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
1245         ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1246         ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1247         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1248                               CCP_PASSTHRU_BYTESWAP_256BIT);
1249         if (ret) {
1250                 cmd->engine_error = cmd_q->cmd_error;
1251                 goto e_key;
1252         }
1253
1254         /* The AES context fits in a single (32-byte) KSB entry and
1255          * for XTS is already in little endian format so no byte swapping
1256          * is needed.
1257          */
1258         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1259                                    CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1260                                    DMA_BIDIRECTIONAL);
1261         if (ret)
1262                 goto e_key;
1263
1264         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1265         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1266                               CCP_PASSTHRU_BYTESWAP_NOOP);
1267         if (ret) {
1268                 cmd->engine_error = cmd_q->cmd_error;
1269                 goto e_ctx;
1270         }
1271
1272         /* Prepare the input and output data workareas. For in-place
1273          * operations we need to set the dma direction to BIDIRECTIONAL
1274          * and copy the src workarea to the dst workarea.
1275          */
1276         if (sg_virt(xts->src) == sg_virt(xts->dst))
1277                 in_place = true;
1278
1279         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1280                             unit_size,
1281                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1282         if (ret)
1283                 goto e_ctx;
1284
1285         if (in_place)
1286                 dst = src;
1287         else {
1288                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1289                                     unit_size, DMA_FROM_DEVICE);
1290                 if (ret)
1291                         goto e_src;
1292         }
1293
1294         /* Send data to the CCP AES engine */
1295         while (src.sg_wa.bytes_left) {
1296                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1297                 if (!src.sg_wa.bytes_left)
1298                         op.eom = 1;
1299
1300                 ret = ccp_perform_xts_aes(&op);
1301                 if (ret) {
1302                         cmd->engine_error = cmd_q->cmd_error;
1303                         goto e_dst;
1304                 }
1305
1306                 ccp_process_data(&src, &dst, &op);
1307         }
1308
1309         /* Retrieve the AES context - convert from LE to BE using
1310          * 32-byte (256-bit) byteswapping
1311          */
1312         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1313                                 CCP_PASSTHRU_BYTESWAP_256BIT);
1314         if (ret) {
1315                 cmd->engine_error = cmd_q->cmd_error;
1316                 goto e_dst;
1317         }
1318
1319         /* ...but we only need AES_BLOCK_SIZE bytes */
1320         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1321         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1322
1323 e_dst:
1324         if (!in_place)
1325                 ccp_free_data(&dst, cmd_q);
1326
1327 e_src:
1328         ccp_free_data(&src, cmd_q);
1329
1330 e_ctx:
1331         ccp_dm_free(&ctx);
1332
1333 e_key:
1334         ccp_dm_free(&key);
1335
1336         return ret;
1337 }
1338
1339 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1340 {
1341         struct ccp_sha_engine *sha = &cmd->u.sha;
1342         struct ccp_dm_workarea ctx;
1343         struct ccp_data src;
1344         struct ccp_op op;
1345         int ret;
1346
1347         if (sha->ctx_len != CCP_SHA_CTXSIZE)
1348                 return -EINVAL;
1349
1350         if (!sha->ctx)
1351                 return -EINVAL;
1352
1353         if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
1354                 return -EINVAL;
1355
1356         if (!sha->src_len) {
1357                 const u8 *sha_zero;
1358
1359                 /* Not final, just return */
1360                 if (!sha->final)
1361                         return 0;
1362
1363                 /* CCP can't do a zero length sha operation so the caller
1364                  * must buffer the data.
1365                  */
1366                 if (sha->msg_bits)
1367                         return -EINVAL;
1368
1369                 /* A sha operation for a message with a total length of zero,
1370                  * return known result.
1371                  */
1372                 switch (sha->type) {
1373                 case CCP_SHA_TYPE_1:
1374                         sha_zero = ccp_sha1_zero;
1375                         break;
1376                 case CCP_SHA_TYPE_224:
1377                         sha_zero = ccp_sha224_zero;
1378                         break;
1379                 case CCP_SHA_TYPE_256:
1380                         sha_zero = ccp_sha256_zero;
1381                         break;
1382                 default:
1383                         return -EINVAL;
1384                 }
1385
1386                 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1387                                          sha->ctx_len, 1);
1388
1389                 return 0;
1390         }
1391
1392         if (!sha->src)
1393                 return -EINVAL;
1394
1395         BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1396
1397         memset(&op, 0, sizeof(op));
1398         op.cmd_q = cmd_q;
1399         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1400         op.ksb_ctx = cmd_q->ksb_ctx;
1401         op.u.sha.type = sha->type;
1402         op.u.sha.msg_bits = sha->msg_bits;
1403
1404         /* The SHA context fits in a single (32-byte) KSB entry and
1405          * must be in little endian format. Use the 256-bit byte swap
1406          * passthru option to convert from big endian to little endian.
1407          */
1408         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1409                                    CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1410                                    DMA_BIDIRECTIONAL);
1411         if (ret)
1412                 return ret;
1413
1414         ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1415         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1416                               CCP_PASSTHRU_BYTESWAP_256BIT);
1417         if (ret) {
1418                 cmd->engine_error = cmd_q->cmd_error;
1419                 goto e_ctx;
1420         }
1421
1422         /* Send data to the CCP SHA engine */
1423         ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1424                             CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1425         if (ret)
1426                 goto e_ctx;
1427
1428         while (src.sg_wa.bytes_left) {
1429                 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1430                 if (sha->final && !src.sg_wa.bytes_left)
1431                         op.eom = 1;
1432
1433                 ret = ccp_perform_sha(&op);
1434                 if (ret) {
1435                         cmd->engine_error = cmd_q->cmd_error;
1436                         goto e_data;
1437                 }
1438
1439                 ccp_process_data(&src, NULL, &op);
1440         }
1441
1442         /* Retrieve the SHA context - convert from LE to BE using
1443          * 32-byte (256-bit) byteswapping to BE
1444          */
1445         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1446                                 CCP_PASSTHRU_BYTESWAP_256BIT);
1447         if (ret) {
1448                 cmd->engine_error = cmd_q->cmd_error;
1449                 goto e_data;
1450         }
1451
1452         ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1453
1454 e_data:
1455         ccp_free_data(&src, cmd_q);
1456
1457 e_ctx:
1458         ccp_dm_free(&ctx);
1459
1460         return ret;
1461 }
1462
1463 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1464 {
1465         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1466         struct ccp_dm_workarea exp, src;
1467         struct ccp_data dst;
1468         struct ccp_op op;
1469         unsigned int ksb_count, i_len, o_len;
1470         int ret;
1471
1472         if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1473                 return -EINVAL;
1474
1475         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1476                 return -EINVAL;
1477
1478         /* The RSA modulus must precede the message being acted upon, so
1479          * it must be copied to a DMA area where the message and the
1480          * modulus can be concatenated.  Therefore the input buffer
1481          * length required is twice the output buffer length (which
1482          * must be a multiple of 256-bits).
1483          */
1484         o_len = ((rsa->key_size + 255) / 256) * 32;
1485         i_len = o_len * 2;
1486
1487         ksb_count = o_len / CCP_KSB_BYTES;
1488
1489         memset(&op, 0, sizeof(op));
1490         op.cmd_q = cmd_q;
1491         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1492         op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1493         if (!op.ksb_key)
1494                 return -EIO;
1495
1496         /* The RSA exponent may span multiple (32-byte) KSB entries and must
1497          * be in little endian format. Reverse copy each 32-byte chunk
1498          * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1499          * and each byte within that chunk and do not perform any byte swap
1500          * operations on the passthru operation.
1501          */
1502         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1503         if (ret)
1504                 goto e_ksb;
1505
1506         ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES,
1507                                 true);
1508         ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1509                               CCP_PASSTHRU_BYTESWAP_NOOP);
1510         if (ret) {
1511                 cmd->engine_error = cmd_q->cmd_error;
1512                 goto e_exp;
1513         }
1514
1515         /* Concatenate the modulus and the message. Both the modulus and
1516          * the operands must be in little endian format.  Since the input
1517          * is in big endian format it must be converted.
1518          */
1519         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1520         if (ret)
1521                 goto e_exp;
1522
1523         ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES,
1524                                 true);
1525         src.address += o_len;   /* Adjust the address for the copy operation */
1526         ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES,
1527                                 true);
1528         src.address -= o_len;   /* Reset the address to original value */
1529
1530         /* Prepare the output area for the operation */
1531         ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1532                             o_len, DMA_FROM_DEVICE);
1533         if (ret)
1534                 goto e_src;
1535
1536         op.soc = 1;
1537         op.src.u.dma.address = src.dma.address;
1538         op.src.u.dma.offset = 0;
1539         op.src.u.dma.length = i_len;
1540         op.dst.u.dma.address = dst.dm_wa.dma.address;
1541         op.dst.u.dma.offset = 0;
1542         op.dst.u.dma.length = o_len;
1543
1544         op.u.rsa.mod_size = rsa->key_size;
1545         op.u.rsa.input_len = i_len;
1546
1547         ret = ccp_perform_rsa(&op);
1548         if (ret) {
1549                 cmd->engine_error = cmd_q->cmd_error;
1550                 goto e_dst;
1551         }
1552
1553         ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1554
1555 e_dst:
1556         ccp_free_data(&dst, cmd_q);
1557
1558 e_src:
1559         ccp_dm_free(&src);
1560
1561 e_exp:
1562         ccp_dm_free(&exp);
1563
1564 e_ksb:
1565         ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1566
1567         return ret;
1568 }
1569
1570 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1571                                 struct ccp_cmd *cmd)
1572 {
1573         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1574         struct ccp_dm_workarea mask;
1575         struct ccp_data src, dst;
1576         struct ccp_op op;
1577         bool in_place = false;
1578         unsigned int i;
1579         int ret;
1580
1581         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1582                 return -EINVAL;
1583
1584         if (!pt->src || !pt->dst)
1585                 return -EINVAL;
1586
1587         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1588                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1589                         return -EINVAL;
1590                 if (!pt->mask)
1591                         return -EINVAL;
1592         }
1593
1594         BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1595
1596         memset(&op, 0, sizeof(op));
1597         op.cmd_q = cmd_q;
1598         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1599
1600         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1601                 /* Load the mask */
1602                 op.ksb_key = cmd_q->ksb_key;
1603
1604                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1605                                            CCP_PASSTHRU_KSB_COUNT *
1606                                            CCP_KSB_BYTES,
1607                                            DMA_TO_DEVICE);
1608                 if (ret)
1609                         return ret;
1610
1611                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1612                 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1613                                       CCP_PASSTHRU_BYTESWAP_NOOP);
1614                 if (ret) {
1615                         cmd->engine_error = cmd_q->cmd_error;
1616                         goto e_mask;
1617                 }
1618         }
1619
1620         /* Prepare the input and output data workareas. For in-place
1621          * operations we need to set the dma direction to BIDIRECTIONAL
1622          * and copy the src workarea to the dst workarea.
1623          */
1624         if (sg_virt(pt->src) == sg_virt(pt->dst))
1625                 in_place = true;
1626
1627         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1628                             CCP_PASSTHRU_MASKSIZE,
1629                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1630         if (ret)
1631                 goto e_mask;
1632
1633         if (in_place)
1634                 dst = src;
1635         else {
1636                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1637                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1638                 if (ret)
1639                         goto e_src;
1640         }
1641
1642         /* Send data to the CCP Passthru engine
1643          *   Because the CCP engine works on a single source and destination
1644          *   dma address at a time, each entry in the source scatterlist
1645          *   (after the dma_map_sg call) must be less than or equal to the
1646          *   (remaining) length in the destination scatterlist entry and the
1647          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1648          */
1649         dst.sg_wa.sg_used = 0;
1650         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1651                 if (!dst.sg_wa.sg ||
1652                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1653                         ret = -EINVAL;
1654                         goto e_dst;
1655                 }
1656
1657                 if (i == src.sg_wa.dma_count) {
1658                         op.eom = 1;
1659                         op.soc = 1;
1660                 }
1661
1662                 op.src.type = CCP_MEMTYPE_SYSTEM;
1663                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1664                 op.src.u.dma.offset = 0;
1665                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1666
1667                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1668                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1669                 op.src.u.dma.offset = dst.sg_wa.sg_used;
1670                 op.src.u.dma.length = op.src.u.dma.length;
1671
1672                 ret = ccp_perform_passthru(&op);
1673                 if (ret) {
1674                         cmd->engine_error = cmd_q->cmd_error;
1675                         goto e_dst;
1676                 }
1677
1678                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1679                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1680                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1681                         dst.sg_wa.sg_used = 0;
1682                 }
1683                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1684         }
1685
1686 e_dst:
1687         if (!in_place)
1688                 ccp_free_data(&dst, cmd_q);
1689
1690 e_src:
1691         ccp_free_data(&src, cmd_q);
1692
1693 e_mask:
1694         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1695                 ccp_dm_free(&mask);
1696
1697         return ret;
1698 }
1699
1700 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1701 {
1702         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1703         struct ccp_dm_workarea src, dst;
1704         struct ccp_op op;
1705         int ret;
1706         u8 *save;
1707
1708         if (!ecc->u.mm.operand_1 ||
1709             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1710                 return -EINVAL;
1711
1712         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1713                 if (!ecc->u.mm.operand_2 ||
1714                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1715                         return -EINVAL;
1716
1717         if (!ecc->u.mm.result ||
1718             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1719                 return -EINVAL;
1720
1721         memset(&op, 0, sizeof(op));
1722         op.cmd_q = cmd_q;
1723         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1724
1725         /* Concatenate the modulus and the operands. Both the modulus and
1726          * the operands must be in little endian format.  Since the input
1727          * is in big endian format it must be converted and placed in a
1728          * fixed length buffer.
1729          */
1730         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1731                                    DMA_TO_DEVICE);
1732         if (ret)
1733                 return ret;
1734
1735         /* Save the workarea address since it is updated in order to perform
1736          * the concatenation
1737          */
1738         save = src.address;
1739
1740         /* Copy the ECC modulus */
1741         ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1742                                 CCP_ECC_OPERAND_SIZE, true);
1743         src.address += CCP_ECC_OPERAND_SIZE;
1744
1745         /* Copy the first operand */
1746         ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1747                                 ecc->u.mm.operand_1_len,
1748                                 CCP_ECC_OPERAND_SIZE, true);
1749         src.address += CCP_ECC_OPERAND_SIZE;
1750
1751         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1752                 /* Copy the second operand */
1753                 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1754                                         ecc->u.mm.operand_2_len,
1755                                         CCP_ECC_OPERAND_SIZE, true);
1756                 src.address += CCP_ECC_OPERAND_SIZE;
1757         }
1758
1759         /* Restore the workarea address */
1760         src.address = save;
1761
1762         /* Prepare the output area for the operation */
1763         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1764                                    DMA_FROM_DEVICE);
1765         if (ret)
1766                 goto e_src;
1767
1768         op.soc = 1;
1769         op.src.u.dma.address = src.dma.address;
1770         op.src.u.dma.offset = 0;
1771         op.src.u.dma.length = src.length;
1772         op.dst.u.dma.address = dst.dma.address;
1773         op.dst.u.dma.offset = 0;
1774         op.dst.u.dma.length = dst.length;
1775
1776         op.u.ecc.function = cmd->u.ecc.function;
1777
1778         ret = ccp_perform_ecc(&op);
1779         if (ret) {
1780                 cmd->engine_error = cmd_q->cmd_error;
1781                 goto e_dst;
1782         }
1783
1784         ecc->ecc_result = le16_to_cpup(
1785                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1786         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1787                 ret = -EIO;
1788                 goto e_dst;
1789         }
1790
1791         /* Save the ECC result */
1792         ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1793
1794 e_dst:
1795         ccp_dm_free(&dst);
1796
1797 e_src:
1798         ccp_dm_free(&src);
1799
1800         return ret;
1801 }
1802
1803 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1804 {
1805         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1806         struct ccp_dm_workarea src, dst;
1807         struct ccp_op op;
1808         int ret;
1809         u8 *save;
1810
1811         if (!ecc->u.pm.point_1.x ||
1812             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1813             !ecc->u.pm.point_1.y ||
1814             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1815                 return -EINVAL;
1816
1817         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1818                 if (!ecc->u.pm.point_2.x ||
1819                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1820                     !ecc->u.pm.point_2.y ||
1821                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1822                         return -EINVAL;
1823         } else {
1824                 if (!ecc->u.pm.domain_a ||
1825                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1826                         return -EINVAL;
1827
1828                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1829                         if (!ecc->u.pm.scalar ||
1830                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1831                                 return -EINVAL;
1832         }
1833
1834         if (!ecc->u.pm.result.x ||
1835             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1836             !ecc->u.pm.result.y ||
1837             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1838                 return -EINVAL;
1839
1840         memset(&op, 0, sizeof(op));
1841         op.cmd_q = cmd_q;
1842         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1843
1844         /* Concatenate the modulus and the operands. Both the modulus and
1845          * the operands must be in little endian format.  Since the input
1846          * is in big endian format it must be converted and placed in a
1847          * fixed length buffer.
1848          */
1849         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1850                                    DMA_TO_DEVICE);
1851         if (ret)
1852                 return ret;
1853
1854         /* Save the workarea address since it is updated in order to perform
1855          * the concatenation
1856          */
1857         save = src.address;
1858
1859         /* Copy the ECC modulus */
1860         ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1861                                 CCP_ECC_OPERAND_SIZE, true);
1862         src.address += CCP_ECC_OPERAND_SIZE;
1863
1864         /* Copy the first point X and Y coordinate */
1865         ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1866                                 ecc->u.pm.point_1.x_len,
1867                                 CCP_ECC_OPERAND_SIZE, true);
1868         src.address += CCP_ECC_OPERAND_SIZE;
1869         ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1870                                 ecc->u.pm.point_1.y_len,
1871                                 CCP_ECC_OPERAND_SIZE, true);
1872         src.address += CCP_ECC_OPERAND_SIZE;
1873
1874         /* Set the first point Z coordianate to 1 */
1875         *(src.address) = 0x01;
1876         src.address += CCP_ECC_OPERAND_SIZE;
1877
1878         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1879                 /* Copy the second point X and Y coordinate */
1880                 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1881                                         ecc->u.pm.point_2.x_len,
1882                                         CCP_ECC_OPERAND_SIZE, true);
1883                 src.address += CCP_ECC_OPERAND_SIZE;
1884                 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1885                                         ecc->u.pm.point_2.y_len,
1886                                         CCP_ECC_OPERAND_SIZE, true);
1887                 src.address += CCP_ECC_OPERAND_SIZE;
1888
1889                 /* Set the second point Z coordianate to 1 */
1890                 *(src.address) = 0x01;
1891                 src.address += CCP_ECC_OPERAND_SIZE;
1892         } else {
1893                 /* Copy the Domain "a" parameter */
1894                 ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1895                                         ecc->u.pm.domain_a_len,
1896                                         CCP_ECC_OPERAND_SIZE, true);
1897                 src.address += CCP_ECC_OPERAND_SIZE;
1898
1899                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1900                         /* Copy the scalar value */
1901                         ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1902                                                 ecc->u.pm.scalar_len,
1903                                                 CCP_ECC_OPERAND_SIZE, true);
1904                         src.address += CCP_ECC_OPERAND_SIZE;
1905                 }
1906         }
1907
1908         /* Restore the workarea address */
1909         src.address = save;
1910
1911         /* Prepare the output area for the operation */
1912         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1913                                    DMA_FROM_DEVICE);
1914         if (ret)
1915                 goto e_src;
1916
1917         op.soc = 1;
1918         op.src.u.dma.address = src.dma.address;
1919         op.src.u.dma.offset = 0;
1920         op.src.u.dma.length = src.length;
1921         op.dst.u.dma.address = dst.dma.address;
1922         op.dst.u.dma.offset = 0;
1923         op.dst.u.dma.length = dst.length;
1924
1925         op.u.ecc.function = cmd->u.ecc.function;
1926
1927         ret = ccp_perform_ecc(&op);
1928         if (ret) {
1929                 cmd->engine_error = cmd_q->cmd_error;
1930                 goto e_dst;
1931         }
1932
1933         ecc->ecc_result = le16_to_cpup(
1934                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1935         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1936                 ret = -EIO;
1937                 goto e_dst;
1938         }
1939
1940         /* Save the workarea address since it is updated as we walk through
1941          * to copy the point math result
1942          */
1943         save = dst.address;
1944
1945         /* Save the ECC result X and Y coordinates */
1946         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1947                                 CCP_ECC_MODULUS_BYTES);
1948         dst.address += CCP_ECC_OUTPUT_SIZE;
1949         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1950                                 CCP_ECC_MODULUS_BYTES);
1951         dst.address += CCP_ECC_OUTPUT_SIZE;
1952
1953         /* Restore the workarea address */
1954         dst.address = save;
1955
1956 e_dst:
1957         ccp_dm_free(&dst);
1958
1959 e_src:
1960         ccp_dm_free(&src);
1961
1962         return ret;
1963 }
1964
1965 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1966 {
1967         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1968
1969         ecc->ecc_result = 0;
1970
1971         if (!ecc->mod ||
1972             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1973                 return -EINVAL;
1974
1975         switch (ecc->function) {
1976         case CCP_ECC_FUNCTION_MMUL_384BIT:
1977         case CCP_ECC_FUNCTION_MADD_384BIT:
1978         case CCP_ECC_FUNCTION_MINV_384BIT:
1979                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1980
1981         case CCP_ECC_FUNCTION_PADD_384BIT:
1982         case CCP_ECC_FUNCTION_PMUL_384BIT:
1983         case CCP_ECC_FUNCTION_PDBL_384BIT:
1984                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1985
1986         default:
1987                 return -EINVAL;
1988         }
1989 }
1990
1991 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1992 {
1993         int ret;
1994
1995         cmd->engine_error = 0;
1996         cmd_q->cmd_error = 0;
1997         cmd_q->int_rcvd = 0;
1998         cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
1999
2000         switch (cmd->engine) {
2001         case CCP_ENGINE_AES:
2002                 ret = ccp_run_aes_cmd(cmd_q, cmd);
2003                 break;
2004         case CCP_ENGINE_XTS_AES_128:
2005                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2006                 break;
2007         case CCP_ENGINE_SHA:
2008                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2009                 break;
2010         case CCP_ENGINE_RSA:
2011                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2012                 break;
2013         case CCP_ENGINE_PASSTHRU:
2014                 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2015                 break;
2016         case CCP_ENGINE_ECC:
2017                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2018                 break;
2019         default:
2020                 ret = -EINVAL;
2021         }
2022
2023         return ret;
2024 }