drivers/crypto/caam/qi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * CAAM/SEC 4.x QI transport/backend driver
   4  * Queue Interface backend functionality
   5  *
   6  * Copyright 2013-2016 Freescale Semiconductor, Inc.
   7  * Copyright 2016-2017, 2019-2020 NXP
   8  */
   9
  10 #include <linux/cpumask.h>
  11 #include <linux/device.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/kernel.h>
  14 #include <linux/kthread.h>
  15 #include <linux/netdevice.h>
  16 #include <linux/slab.h>
  17 #include <linux/string.h>
  18 #include <soc/fsl/qman.h>
  19
  20 #include "debugfs.h"
  21 #include "regs.h"
  22 #include "qi.h"
  23 #include "desc.h"
  24 #include "intern.h"
  25 #include "desc_constr.h"
  26
  27 #define PREHDR_RSLS_SHIFT       31
  28 #define PREHDR_ABS              BIT(25)
  29
  30 /*
  31  * Use a reasonable backlog of frames (per CPU) as congestion threshold,
  32  * so that resources used by the in-flight buffers do not become a memory hog.
  33  */
  34 #define MAX_RSP_FQ_BACKLOG_PER_CPU      256
  35
  36 #define CAAM_QI_ENQUEUE_RETRIES 10000
  37
  38 #define CAAM_NAPI_WEIGHT        63
  39
  40 /*
  41  * caam_napi - struct holding CAAM NAPI-related params
  42  * @irqtask: IRQ task for QI backend
  43  * @p: QMan portal
  44  */
  45 struct caam_napi {
  46         struct napi_struct irqtask;
  47         struct qman_portal *p;
  48 };
  49
  50 /*
  51  * caam_qi_pcpu_priv - percpu private data structure to main list of pending
  52  *                     responses expected on each cpu.
  53  * @caam_napi: CAAM NAPI params
  54  * @net_dev: netdev used by NAPI
  55  * @rsp_fq: response FQ from CAAM
  56  */
  57 struct caam_qi_pcpu_priv {
  58         struct caam_napi caam_napi;
  59         struct net_device net_dev;
  60         struct qman_fq *rsp_fq;
  61 } ____cacheline_aligned;
  62
  63 static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
  64 static DEFINE_PER_CPU(int, last_cpu);
  65
  66 /*
  67  * caam_qi_priv - CAAM QI backend private params
  68  * @cgr: QMan congestion group
  69  */
  70 struct caam_qi_priv {
  71         struct qman_cgr cgr;
  72 };
  73
  74 static struct caam_qi_priv qipriv ____cacheline_aligned;
  75
  76 /*
  77  * This is written by only one core - the one that initialized the CGR - and
  78  * read by multiple cores (all the others).
  79  */
  80 bool caam_congested __read_mostly;
  81 EXPORT_SYMBOL(caam_congested);
  82
  83 /*
  84  * This is a cache of buffers, from which the users of CAAM QI driver
  85  * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
  86  * doing malloc on the hotpath.
  87  * NOTE: A more elegant solution would be to have some headroom in the frames
  88  *       being processed. This could be added by the dpaa-ethernet driver.
  89  *       This would pose a problem for userspace application processing which
  90  *       cannot know of this limitation. So for now, this will work.
  91  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  92  */
  93 static struct kmem_cache *qi_cache;
  94
  95 static void *caam_iova_to_virt(struct iommu_domain *domain,
  96                                dma_addr_t iova_addr)
  97 {
  98         phys_addr_t phys_addr;
  99
 100         phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
 101
 102         return phys_to_virt(phys_addr);
 103 }
 104
 105 int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
 106 {
 107         struct qm_fd fd;
 108         dma_addr_t addr;
 109         int ret;
 110         int num_retries = 0;
 111
 112         qm_fd_clear_fd(&fd);
 113         qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1]));
 114
 115         addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt),
 116                               DMA_BIDIRECTIONAL);
 117         if (dma_mapping_error(qidev, addr)) {
 118                 dev_err(qidev, "DMA mapping error for QI enqueue request\n");
 119                 return -EIO;
 120         }
 121         qm_fd_addr_set64(&fd, addr);
 122
 123         do {
 124                 ret = qman_enqueue(req->drv_ctx->req_fq, &fd);
 125                 if (likely(!ret)) {
 126                         refcount_inc(&req->drv_ctx->refcnt);
 127                         return 0;
 128                 }
 129
 130                 if (ret != -EBUSY)
 131                         break;
 132                 num_retries++;
 133         } while (num_retries < CAAM_QI_ENQUEUE_RETRIES);
 134
 135         dev_err(qidev, "qman_enqueue failed: %d\n", ret);
 136
 137         return ret;
 138 }
 139 EXPORT_SYMBOL(caam_qi_enqueue);
 140
 141 static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
 142                            const union qm_mr_entry *msg)
 143 {
 144         const struct qm_fd *fd;
 145         struct caam_drv_req *drv_req;
 146         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
 147         struct caam_drv_private *priv = dev_get_drvdata(qidev);
 148
 149         fd = &msg->ern.fd;
 150
 151         drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
 152         if (!drv_req) {
 153                 dev_err(qidev,
 154                         "Can't find original request for CAAM response\n");
 155                 return;
 156         }
 157
 158         refcount_dec(&drv_req->drv_ctx->refcnt);
 159
 160         if (qm_fd_get_format(fd) != qm_fd_compound) {
 161                 dev_err(qidev, "Non-compound FD from CAAM\n");
 162                 return;
 163         }
 164
 165         dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
 166                          sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
 167
 168         if (fd->status)
 169                 drv_req->cbk(drv_req, be32_to_cpu(fd->status));
 170         else
 171                 drv_req->cbk(drv_req, JRSTA_SSRC_QI);
 172 }
 173
 174 static struct qman_fq *create_caam_req_fq(struct device *qidev,
 175                                           struct qman_fq *rsp_fq,
 176                                           dma_addr_t hwdesc,
 177                                           int fq_sched_flag)
 178 {
 179         int ret;
 180         struct qman_fq *req_fq;
 181         struct qm_mcc_initfq opts;
 182
 183         req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC);
 184         if (!req_fq)
 185                 return ERR_PTR(-ENOMEM);
 186
 187         req_fq->cb.ern = caam_fq_ern_cb;
 188         req_fq->cb.fqs = NULL;
 189
 190         ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
 191                                 QMAN_FQ_FLAG_TO_DCPORTAL, req_fq);
 192         if (ret) {
 193                 dev_err(qidev, "Failed to create session req FQ\n");
 194                 goto create_req_fq_fail;
 195         }
 196
 197         memset(&opts, 0, sizeof(opts));
 198         opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
 199                                    QM_INITFQ_WE_CONTEXTB |
 200                                    QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
 201         opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
 202         qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2);
 203         opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq));
 204         qm_fqd_context_a_set64(&opts.fqd, hwdesc);
 205         opts.fqd.cgid = qipriv.cgr.cgrid;
 206
 207         ret = qman_init_fq(req_fq, fq_sched_flag, &opts);
 208         if (ret) {
 209                 dev_err(qidev, "Failed to init session req FQ\n");
 210                 goto init_req_fq_fail;
 211         }
 212
 213         dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
 214                 smp_processor_id());
 215         return req_fq;
 216
 217 init_req_fq_fail:
 218         qman_destroy_fq(req_fq);
 219 create_req_fq_fail:
 220         kfree(req_fq);
 221         return ERR_PTR(ret);
 222 }
 223
 224 static int empty_retired_fq(struct device *qidev, struct qman_fq *fq)
 225 {
 226         int ret;
 227
 228         ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT |
 229                                     QMAN_VOLATILE_FLAG_FINISH,
 230                                     QM_VDQCR_PRECEDENCE_VDQCR |
 231                                     QM_VDQCR_NUMFRAMES_TILLEMPTY);
 232         if (ret) {
 233                 dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid);
 234                 return ret;
 235         }
 236
 237         do {
 238                 struct qman_portal *p;
 239
 240                 p = qman_get_affine_portal(smp_processor_id());
 241                 qman_p_poll_dqrr(p, 16);
 242         } while (fq->flags & QMAN_FQ_STATE_NE);
 243
 244         return 0;
 245 }
 246
 247 static int kill_fq(struct device *qidev, struct qman_fq *fq)
 248 {
 249         u32 flags;
 250         int ret;
 251
 252         ret = qman_retire_fq(fq, &flags);
 253         if (ret < 0) {
 254                 dev_err(qidev, "qman_retire_fq failed: %d\n", ret);
 255                 return ret;
 256         }
 257
 258         if (!ret)
 259                 goto empty_fq;
 260
 261         /* Async FQ retirement condition */
 262         if (ret == 1) {
 263                 /* Retry till FQ gets in retired state */
 264                 do {
 265                         msleep(20);
 266                 } while (fq->state != qman_fq_state_retired);
 267
 268                 WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS);
 269                 WARN_ON(fq->flags & QMAN_FQ_STATE_ORL);
 270         }
 271
 272 empty_fq:
 273         if (fq->flags & QMAN_FQ_STATE_NE) {
 274                 ret = empty_retired_fq(qidev, fq);
 275                 if (ret) {
 276                         dev_err(qidev, "empty_retired_fq fail for FQ: %u\n",
 277                                 fq->fqid);
 278                         return ret;
 279                 }
 280         }
 281
 282         ret = qman_oos_fq(fq);
 283         if (ret)
 284                 dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
 285
 286         qman_destroy_fq(fq);
 287         kfree(fq);
 288
 289         return ret;
 290 }
 291
 292 static int empty_caam_fq(struct qman_fq *fq, struct caam_drv_ctx *drv_ctx)
 293 {
 294         int ret;
 295         int retries = 10;
 296         struct qm_mcr_queryfq_np np;
 297
 298         /* Wait till the older CAAM FQ get empty */
 299         do {
 300                 ret = qman_query_fq_np(fq, &np);
 301                 if (ret)
 302                         return ret;
 303
 304                 if (!qm_mcr_np_get(&np, frm_cnt))
 305                         break;
 306
 307                 msleep(20);
 308         } while (1);
 309
 310         /* Wait until pending jobs from this FQ are processed by CAAM */
 311         do {
 312                 if (refcount_read(&drv_ctx->refcnt) == 1)
 313                         break;
 314
 315                 msleep(20);
 316         } while (--retries);
 317
 318         if (!retries)
 319                 dev_warn_once(drv_ctx->qidev, "%d frames from FQID %u still pending in CAAM\n",
 320                               refcount_read(&drv_ctx->refcnt), fq->fqid);
 321
 322         return 0;
 323 }
 324
 325 int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
 326 {
 327         int ret;
 328         u32 num_words;
 329         struct qman_fq *new_fq, *old_fq;
 330         struct device *qidev = drv_ctx->qidev;
 331
 332         num_words = desc_len(sh_desc);
 333         if (num_words > MAX_SDLEN) {
 334                 dev_err(qidev, "Invalid descriptor len: %d words\n", num_words);
 335                 return -EINVAL;
 336         }
 337
 338         /* Note down older req FQ */
 339         old_fq = drv_ctx->req_fq;
 340
 341         /* Create a new req FQ in parked state */
 342         new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq,
 343                                     drv_ctx->context_a, 0);
 344         if (IS_ERR(new_fq)) {
 345                 dev_err(qidev, "FQ allocation for shdesc update failed\n");
 346                 return PTR_ERR(new_fq);
 347         }
 348
 349         /* Hook up new FQ to context so that new requests keep queuing */
 350         drv_ctx->req_fq = new_fq;
 351
 352         /* Empty and remove the older FQ */
 353         ret = empty_caam_fq(old_fq, drv_ctx);
 354         if (ret) {
 355                 dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret);
 356
 357                 /* We can revert to older FQ */
 358                 drv_ctx->req_fq = old_fq;
 359
 360                 if (kill_fq(qidev, new_fq))
 361                         dev_warn(qidev, "New CAAM FQ kill failed\n");
 362
 363                 return ret;
 364         }
 365
 366         /*
 367          * Re-initialise pre-header. Set RSLS and SDLEN.
 368          * Update the shared descriptor for driver context.
 369          */
 370         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
 371                                            num_words);
 372         drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
 373         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
 374         dma_sync_single_for_device(qidev, drv_ctx->context_a,
 375                                    sizeof(drv_ctx->sh_desc) +
 376                                    sizeof(drv_ctx->prehdr),
 377                                    DMA_BIDIRECTIONAL);
 378
 379         /* Put the new FQ in scheduled state */
 380         ret = qman_schedule_fq(new_fq);
 381         if (ret) {
 382                 dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret);
 383
 384                 /*
 385                  * We can kill new FQ and revert to old FQ.
 386                  * Since the desc is already modified, it is success case
 387                  */
 388
 389                 drv_ctx->req_fq = old_fq;
 390
 391                 if (kill_fq(qidev, new_fq))
 392                         dev_warn(qidev, "New CAAM FQ kill failed\n");
 393         } else if (kill_fq(qidev, old_fq)) {
 394                 dev_warn(qidev, "Old CAAM FQ kill failed\n");
 395         }
 396
 397         return 0;
 398 }
 399 EXPORT_SYMBOL(caam_drv_ctx_update);
 400
 401 struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
 402                                        int *cpu,
 403                                        u32 *sh_desc)
 404 {
 405         size_t size;
 406         u32 num_words;
 407         dma_addr_t hwdesc;
 408         struct caam_drv_ctx *drv_ctx;
 409         const cpumask_t *cpus = qman_affine_cpus();
 410
 411         num_words = desc_len(sh_desc);
 412         if (num_words > MAX_SDLEN) {
 413                 dev_err(qidev, "Invalid descriptor len: %d words\n",
 414                         num_words);
 415                 return ERR_PTR(-EINVAL);
 416         }
 417
 418         drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC);
 419         if (!drv_ctx)
 420                 return ERR_PTR(-ENOMEM);
 421
 422         /*
 423          * Initialise pre-header - set RSLS and SDLEN - and shared descriptor
 424          * and dma-map them.
 425          */
 426         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
 427                                            num_words);
 428         drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
 429         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
 430         size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
 431         hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
 432                                 DMA_BIDIRECTIONAL);
 433         if (dma_mapping_error(qidev, hwdesc)) {
 434                 dev_err(qidev, "DMA map error for preheader + shdesc\n");
 435                 kfree(drv_ctx);
 436                 return ERR_PTR(-ENOMEM);
 437         }
 438         drv_ctx->context_a = hwdesc;
 439
 440         /* If given CPU does not own the portal, choose another one that does */
 441         if (!cpumask_test_cpu(*cpu, cpus)) {
 442                 int *pcpu = &get_cpu_var(last_cpu);
 443
 444                 *pcpu = cpumask_next(*pcpu, cpus);
 445                 if (*pcpu >= nr_cpu_ids)
 446                         *pcpu = cpumask_first(cpus);
 447                 *cpu = *pcpu;
 448
 449                 put_cpu_var(last_cpu);
 450         }
 451         drv_ctx->cpu = *cpu;
 452
 453         /* Find response FQ hooked with this CPU */
 454         drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu);
 455
 456         /* Attach request FQ */
 457         drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc,
 458                                              QMAN_INITFQ_FLAG_SCHED);
 459         if (IS_ERR(drv_ctx->req_fq)) {
 460                 dev_err(qidev, "create_caam_req_fq failed\n");
 461                 dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL);
 462                 kfree(drv_ctx);
 463                 return ERR_PTR(-ENOMEM);
 464         }
 465
 466         /* init reference counter used to track references to request FQ */
 467         refcount_set(&drv_ctx->refcnt, 1);
 468
 469         drv_ctx->qidev = qidev;
 470         return drv_ctx;
 471 }
 472 EXPORT_SYMBOL(caam_drv_ctx_init);
 473
 474 void *qi_cache_alloc(gfp_t flags)
 475 {
 476         return kmem_cache_alloc(qi_cache, flags);
 477 }
 478 EXPORT_SYMBOL(qi_cache_alloc);
 479
 480 void qi_cache_free(void *obj)
 481 {
 482         kmem_cache_free(qi_cache, obj);
 483 }
 484 EXPORT_SYMBOL(qi_cache_free);
 485
 486 static int caam_qi_poll(struct napi_struct *napi, int budget)
 487 {
 488         struct caam_napi *np = container_of(napi, struct caam_napi, irqtask);
 489
 490         int cleaned = qman_p_poll_dqrr(np->p, budget);
 491
 492         if (cleaned < budget) {
 493                 napi_complete(napi);
 494                 qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
 495         }
 496
 497         return cleaned;
 498 }
 499
 500 void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx)
 501 {
 502         if (IS_ERR_OR_NULL(drv_ctx))
 503                 return;
 504
 505         /* Remove request FQ */
 506         if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq))
 507                 dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n");
 508
 509         dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a,
 510                          sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr),
 511                          DMA_BIDIRECTIONAL);
 512         kfree(drv_ctx);
 513 }
 514 EXPORT_SYMBOL(caam_drv_ctx_rel);
 515
 516 static void caam_qi_shutdown(void *data)
 517 {
 518         int i;
 519         struct device *qidev = data;
 520         struct caam_qi_priv *priv = &qipriv;
 521         const cpumask_t *cpus = qman_affine_cpus();
 522
 523         for_each_cpu(i, cpus) {
 524                 struct napi_struct *irqtask;
 525
 526                 irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask;
 527                 napi_disable(irqtask);
 528                 netif_napi_del(irqtask);
 529
 530                 if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i)))
 531                         dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
 532         }
 533
 534         qman_delete_cgr_safe(&priv->cgr);
 535         qman_release_cgrid(priv->cgr.cgrid);
 536
 537         kmem_cache_destroy(qi_cache);
 538 }
 539
 540 static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
 541 {
 542         caam_congested = congested;
 543
 544         if (congested) {
 545                 caam_debugfs_qi_congested();
 546
 547                 pr_debug_ratelimited("CAAM entered congestion\n");
 548
 549         } else {
 550                 pr_debug_ratelimited("CAAM exited congestion\n");
 551         }
 552 }
 553
 554 static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np,
 555                                  bool sched_napi)
 556 {
 557         if (sched_napi) {
 558                 /* Disable QMan IRQ source and invoke NAPI */
 559                 qman_p_irqsource_remove(p, QM_PIRQ_DQRI);
 560                 np->p = p;
 561                 napi_schedule(&np->irqtask);
 562                 return 1;
 563         }
 564         return 0;
 565 }
 566
 567 static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
 568                                                     struct qman_fq *rsp_fq,
 569                                                     const struct qm_dqrr_entry *dqrr,
 570                                                     bool sched_napi)
 571 {
 572         struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi);
 573         struct caam_drv_req *drv_req;
 574         const struct qm_fd *fd;
 575         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
 576         struct caam_drv_private *priv = dev_get_drvdata(qidev);
 577         u32 status;
 578
 579         if (caam_qi_napi_schedule(p, caam_napi, sched_napi))
 580                 return qman_cb_dqrr_stop;
 581
 582         fd = &dqrr->fd;
 583
 584         drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
 585         if (unlikely(!drv_req)) {
 586                 dev_err(qidev,
 587                         "Can't find original request for caam response\n");
 588                 return qman_cb_dqrr_consume;
 589         }
 590
 591         refcount_dec(&drv_req->drv_ctx->refcnt);
 592
 593         status = be32_to_cpu(fd->status);
 594         if (unlikely(status)) {
 595                 u32 ssrc = status & JRSTA_SSRC_MASK;
 596                 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
 597
 598                 if (ssrc != JRSTA_SSRC_CCB_ERROR ||
 599                     err_id != JRSTA_CCBERR_ERRID_ICVCHK)
 600                         dev_err_ratelimited(qidev,
 601                                             "Error: %#x in CAAM response FD\n",
 602                                             status);
 603         }
 604
 605         if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
 606                 dev_err(qidev, "Non-compound FD from CAAM\n");
 607                 return qman_cb_dqrr_consume;
 608         }
 609
 610         dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
 611                          sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
 612
 613         drv_req->cbk(drv_req, status);
 614         return qman_cb_dqrr_consume;
 615 }
 616
 617 static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
 618 {
 619         struct qm_mcc_initfq opts;
 620         struct qman_fq *fq;
 621         int ret;
 622
 623         fq = kzalloc(sizeof(*fq), GFP_KERNEL);
 624         if (!fq)
 625                 return -ENOMEM;
 626
 627         fq->cb.dqrr = caam_rsp_fq_dqrr_cb;
 628
 629         ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE |
 630                              QMAN_FQ_FLAG_DYNAMIC_FQID, fq);
 631         if (ret) {
 632                 dev_err(qidev, "Rsp FQ create failed\n");
 633                 kfree(fq);
 634                 return -ENODEV;
 635         }
 636
 637         memset(&opts, 0, sizeof(opts));
 638         opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
 639                                    QM_INITFQ_WE_CONTEXTB |
 640                                    QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
 641         opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING |
 642                                        QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
 643         qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3);
 644         opts.fqd.cgid = qipriv.cgr.cgrid;
 645         opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX |
 646                                                 QM_STASHING_EXCL_DATA;
 647         qm_fqd_set_stashing(&opts.fqd, 0, 1, 1);
 648
 649         ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
 650         if (ret) {
 651                 dev_err(qidev, "Rsp FQ init failed\n");
 652                 kfree(fq);
 653                 return -ENODEV;
 654         }
 655
 656         per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq;
 657
 658         dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
 659         return 0;
 660 }
 661
 662 static int init_cgr(struct device *qidev)
 663 {
 664         int ret;
 665         struct qm_mcc_initcgr opts;
 666         const u64 val = (u64)cpumask_weight(qman_affine_cpus()) *
 667                         MAX_RSP_FQ_BACKLOG_PER_CPU;
 668
 669         ret = qman_alloc_cgrid(&qipriv.cgr.cgrid);
 670         if (ret) {
 671                 dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret);
 672                 return ret;
 673         }
 674
 675         qipriv.cgr.cb = cgr_cb;
 676         memset(&opts, 0, sizeof(opts));
 677         opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES |
 678                                    QM_CGR_WE_MODE);
 679         opts.cgr.cscn_en = QM_CGR_EN;
 680         opts.cgr.mode = QMAN_CGR_MODE_FRAME;
 681         qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1);
 682
 683         ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts);
 684         if (ret) {
 685                 dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret,
 686                         qipriv.cgr.cgrid);
 687                 return ret;
 688         }
 689
 690         dev_dbg(qidev, "Congestion threshold set to %llu\n", val);
 691         return 0;
 692 }
 693
 694 static int alloc_rsp_fqs(struct device *qidev)
 695 {
 696         int ret, i;
 697         const cpumask_t *cpus = qman_affine_cpus();
 698
 699         /*Now create response FQs*/
 700         for_each_cpu(i, cpus) {
 701                 ret = alloc_rsp_fq_cpu(qidev, i);
 702                 if (ret) {
 703                         dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i);
 704                         return ret;
 705                 }
 706         }
 707
 708         return 0;
 709 }
 710
 711 static void free_rsp_fqs(void)
 712 {
 713         int i;
 714         const cpumask_t *cpus = qman_affine_cpus();
 715
 716         for_each_cpu(i, cpus)
 717                 kfree(per_cpu(pcpu_qipriv.rsp_fq, i));
 718 }
 719
 720 int caam_qi_init(struct platform_device *caam_pdev)
 721 {
 722         int err, i;
 723         struct device *ctrldev = &caam_pdev->dev, *qidev;
 724         struct caam_drv_private *ctrlpriv;
 725         const cpumask_t *cpus = qman_affine_cpus();
 726
 727         ctrlpriv = dev_get_drvdata(ctrldev);
 728         qidev = ctrldev;
 729
 730         /* Initialize the congestion detection */
 731         err = init_cgr(qidev);
 732         if (err) {
 733                 dev_err(qidev, "CGR initialization failed: %d\n", err);
 734                 return err;
 735         }
 736
 737         /* Initialise response FQs */
 738         err = alloc_rsp_fqs(qidev);
 739         if (err) {
 740                 dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
 741                 free_rsp_fqs();
 742                 return err;
 743         }
 744
 745         /*
 746          * Enable the NAPI contexts on each of the core which has an affine
 747          * portal.
 748          */
 749         for_each_cpu(i, cpus) {
 750                 struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i);
 751                 struct caam_napi *caam_napi = &priv->caam_napi;
 752                 struct napi_struct *irqtask = &caam_napi->irqtask;
 753                 struct net_device *net_dev = &priv->net_dev;
 754
 755                 net_dev->dev = *qidev;
 756                 INIT_LIST_HEAD(&net_dev->napi_list);
 757
 758                 netif_napi_add_tx_weight(net_dev, irqtask, caam_qi_poll,
 759                                          CAAM_NAPI_WEIGHT);
 760
 761                 napi_enable(irqtask);
 762         }
 763
 764         qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE,
 765                                      dma_get_cache_alignment(), 0, NULL);
 766         if (!qi_cache) {
 767                 dev_err(qidev, "Can't allocate CAAM cache\n");
 768                 free_rsp_fqs();
 769                 return -ENOMEM;
 770         }
 771
 772         caam_debugfs_qi_init(ctrlpriv);
 773
 774         err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv);
 775         if (err)
 776                 return err;
 777
 778         dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
 779         return 0;
 780 }