drivers/infiniband/hw/bnxt_re/qplib_res.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: QPLib resource manager
  37  */
  38
  39 #define dev_fmt(fmt) "QPLIB: " fmt
  40
  41 #include <linux/spinlock.h>
  42 #include <linux/pci.h>
  43 #include <linux/interrupt.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/dma-mapping.h>
  46 #include <linux/if_vlan.h>
  47 #include <linux/vmalloc.h>
  48 #include <rdma/ib_verbs.h>
  49 #include <rdma/ib_umem.h>
  50
  51 #include "roce_hsi.h"
  52 #include "qplib_res.h"
  53 #include "qplib_sp.h"
  54 #include "qplib_rcfw.h"
  55
  56 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
  57                                       struct bnxt_qplib_stats *stats);
  58 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
  59                                       struct bnxt_qplib_chip_ctx *cctx,
  60                                       struct bnxt_qplib_stats *stats);
  61
  62 /* PBL */
  63 static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
  64                        bool is_umem)
  65 {
  66         struct pci_dev *pdev = res->pdev;
  67         int i;
  68
  69         if (!is_umem) {
  70                 for (i = 0; i < pbl->pg_count; i++) {
  71                         if (pbl->pg_arr[i])
  72                                 dma_free_coherent(&pdev->dev, pbl->pg_size,
  73                                                   (void *)((unsigned long)
  74                                                    pbl->pg_arr[i] &
  75                                                   PAGE_MASK),
  76                                                   pbl->pg_map_arr[i]);
  77                         else
  78                                 dev_warn(&pdev->dev,
  79                                          "PBL free pg_arr[%d] empty?!\n", i);
  80                         pbl->pg_arr[i] = NULL;
  81                 }
  82         }
  83         vfree(pbl->pg_arr);
  84         pbl->pg_arr = NULL;
  85         vfree(pbl->pg_map_arr);
  86         pbl->pg_map_arr = NULL;
  87         pbl->pg_count = 0;
  88         pbl->pg_size = 0;
  89 }
  90
  91 static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
  92                                            struct bnxt_qplib_sg_info *sginfo)
  93 {
  94         struct ib_block_iter biter;
  95         int i = 0;
  96
  97         rdma_umem_for_each_dma_block(sginfo->umem, &biter, sginfo->pgsize) {
  98                 pbl->pg_map_arr[i] = rdma_block_iter_dma_address(&biter);
  99                 pbl->pg_arr[i] = NULL;
 100                 pbl->pg_count++;
 101                 i++;
 102         }
 103 }
 104
 105 static int __alloc_pbl(struct bnxt_qplib_res *res,
 106                        struct bnxt_qplib_pbl *pbl,
 107                        struct bnxt_qplib_sg_info *sginfo)
 108 {
 109         struct pci_dev *pdev = res->pdev;
 110         bool is_umem = false;
 111         u32 pages;
 112         int i;
 113
 114         if (sginfo->nopte)
 115                 return 0;
 116         if (sginfo->umem)
 117                 pages = ib_umem_num_dma_blocks(sginfo->umem, sginfo->pgsize);
 118         else
 119                 pages = sginfo->npages;
 120         /* page ptr arrays */
 121         pbl->pg_arr = vmalloc(pages * sizeof(void *));
 122         if (!pbl->pg_arr)
 123                 return -ENOMEM;
 124
 125         pbl->pg_map_arr = vmalloc(pages * sizeof(dma_addr_t));
 126         if (!pbl->pg_map_arr) {
 127                 vfree(pbl->pg_arr);
 128                 pbl->pg_arr = NULL;
 129                 return -ENOMEM;
 130         }
 131         pbl->pg_count = 0;
 132         pbl->pg_size = sginfo->pgsize;
 133
 134         if (!sginfo->umem) {
 135                 for (i = 0; i < pages; i++) {
 136                         pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
 137                                                             pbl->pg_size,
 138                                                             &pbl->pg_map_arr[i],
 139                                                             GFP_KERNEL);
 140                         if (!pbl->pg_arr[i])
 141                                 goto fail;
 142                         pbl->pg_count++;
 143                 }
 144         } else {
 145                 is_umem = true;
 146                 bnxt_qplib_fill_user_dma_pages(pbl, sginfo);
 147         }
 148
 149         return 0;
 150 fail:
 151         __free_pbl(res, pbl, is_umem);
 152         return -ENOMEM;
 153 }
 154
 155 /* HWQ */
 156 void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
 157                          struct bnxt_qplib_hwq *hwq)
 158 {
 159         int i;
 160
 161         if (!hwq->max_elements)
 162                 return;
 163         if (hwq->level >= PBL_LVL_MAX)
 164                 return;
 165
 166         for (i = 0; i < hwq->level + 1; i++) {
 167                 if (i == hwq->level)
 168                         __free_pbl(res, &hwq->pbl[i], hwq->is_user);
 169                 else
 170                         __free_pbl(res, &hwq->pbl[i], false);
 171         }
 172
 173         hwq->level = PBL_LVL_MAX;
 174         hwq->max_elements = 0;
 175         hwq->element_size = 0;
 176         hwq->prod = 0;
 177         hwq->cons = 0;
 178         hwq->cp_bit = 0;
 179 }
 180
 181 /* All HWQs are power of 2 in size */
 182
 183 int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
 184                               struct bnxt_qplib_hwq_attr *hwq_attr)
 185 {
 186         u32 npages, aux_slots, pg_size, aux_pages = 0, aux_size = 0;
 187         struct bnxt_qplib_sg_info sginfo = {};
 188         u32 depth, stride, npbl, npde;
 189         dma_addr_t *src_phys_ptr, **dst_virt_ptr;
 190         struct bnxt_qplib_res *res;
 191         struct pci_dev *pdev;
 192         int i, rc, lvl;
 193
 194         res = hwq_attr->res;
 195         pdev = res->pdev;
 196         pg_size = hwq_attr->sginfo->pgsize;
 197         hwq->level = PBL_LVL_MAX;
 198
 199         depth = roundup_pow_of_two(hwq_attr->depth);
 200         stride = roundup_pow_of_two(hwq_attr->stride);
 201         if (hwq_attr->aux_depth) {
 202                 aux_slots = hwq_attr->aux_depth;
 203                 aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
 204                 aux_pages = (aux_slots * aux_size) / pg_size;
 205                 if ((aux_slots * aux_size) % pg_size)
 206                         aux_pages++;
 207         }
 208
 209         if (!hwq_attr->sginfo->umem) {
 210                 hwq->is_user = false;
 211                 npages = (depth * stride) / pg_size + aux_pages;
 212                 if ((depth * stride) % pg_size)
 213                         npages++;
 214                 if (!npages)
 215                         return -EINVAL;
 216                 hwq_attr->sginfo->npages = npages;
 217         } else {
 218                 unsigned long sginfo_num_pages = ib_umem_num_dma_blocks(
 219                         hwq_attr->sginfo->umem, hwq_attr->sginfo->pgsize);
 220
 221                 hwq->is_user = true;
 222                 npages = sginfo_num_pages;
 223                 npages = (npages * PAGE_SIZE) /
 224                           BIT_ULL(hwq_attr->sginfo->pgshft);
 225                 if ((sginfo_num_pages * PAGE_SIZE) %
 226                      BIT_ULL(hwq_attr->sginfo->pgshft))
 227                         if (!npages)
 228                                 npages++;
 229         }
 230
 231         if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) {
 232                 /* This request is Level 0, map PTE */
 233                 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
 234                 if (rc)
 235                         goto fail;
 236                 hwq->level = PBL_LVL_0;
 237                 goto done;
 238         }
 239
 240         if (npages >= MAX_PBL_LVL_0_PGS) {
 241                 if (npages > MAX_PBL_LVL_1_PGS) {
 242                         u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
 243                                     0 : PTU_PTE_VALID;
 244                         /* 2 levels of indirection */
 245                         npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
 246                         if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
 247                                 npbl++;
 248                         npde = npbl >> MAX_PDL_LVL_SHIFT;
 249                         if (npbl % BIT(MAX_PDL_LVL_SHIFT))
 250                                 npde++;
 251                         /* Alloc PDE pages */
 252                         sginfo.pgsize = npde * pg_size;
 253                         sginfo.npages = 1;
 254                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
 255
 256                         /* Alloc PBL pages */
 257                         sginfo.npages = npbl;
 258                         sginfo.pgsize = PAGE_SIZE;
 259                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
 260                         if (rc)
 261                                 goto fail;
 262                         /* Fill PDL with PBL page pointers */
 263                         dst_virt_ptr =
 264                                 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
 265                         src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
 266                         if (hwq_attr->type == HWQ_TYPE_MR) {
 267                         /* For MR it is expected that we supply only 1 contigous
 268                          * page i.e only 1 entry in the PDL that will contain
 269                          * all the PBLs for the user supplied memory region
 270                          */
 271                                 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
 272                                      i++)
 273                                         dst_virt_ptr[0][i] = src_phys_ptr[i] |
 274                                                 flag;
 275                         } else {
 276                                 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
 277                                      i++)
 278                                         dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 279                                                 src_phys_ptr[i] |
 280                                                 PTU_PDE_VALID;
 281                         }
 282                         /* Alloc or init PTEs */
 283                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
 284                                          hwq_attr->sginfo);
 285                         if (rc)
 286                                 goto fail;
 287                         hwq->level = PBL_LVL_2;
 288                         if (hwq_attr->sginfo->nopte)
 289                                 goto done;
 290                         /* Fill PBLs with PTE pointers */
 291                         dst_virt_ptr =
 292                                 (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
 293                         src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
 294                         for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
 295                                 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 296                                         src_phys_ptr[i] | PTU_PTE_VALID;
 297                         }
 298                         if (hwq_attr->type == HWQ_TYPE_QUEUE) {
 299                                 /* Find the last pg of the size */
 300                                 i = hwq->pbl[PBL_LVL_2].pg_count;
 301                                 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
 302                                                                   PTU_PTE_LAST;
 303                                 if (i > 1)
 304                                         dst_virt_ptr[PTR_PG(i - 2)]
 305                                                     [PTR_IDX(i - 2)] |=
 306                                                     PTU_PTE_NEXT_TO_LAST;
 307                         }
 308                 } else { /* pages < 512 npbl = 1, npde = 0 */
 309                         u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
 310                                     0 : PTU_PTE_VALID;
 311
 312                         /* 1 level of indirection */
 313                         npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
 314                         if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
 315                                 npbl++;
 316                         sginfo.npages = npbl;
 317                         sginfo.pgsize = PAGE_SIZE;
 318                         /* Alloc PBL page */
 319                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
 320                         if (rc)
 321                                 goto fail;
 322                         /* Alloc or init  PTEs */
 323                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
 324                                          hwq_attr->sginfo);
 325                         if (rc)
 326                                 goto fail;
 327                         hwq->level = PBL_LVL_1;
 328                         if (hwq_attr->sginfo->nopte)
 329                                 goto done;
 330                         /* Fill PBL with PTE pointers */
 331                         dst_virt_ptr =
 332                                 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
 333                         src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
 334                         for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
 335                                 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 336                                         src_phys_ptr[i] | flag;
 337                         if (hwq_attr->type == HWQ_TYPE_QUEUE) {
 338                                 /* Find the last pg of the size */
 339                                 i = hwq->pbl[PBL_LVL_1].pg_count;
 340                                 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
 341                                                                   PTU_PTE_LAST;
 342                                 if (i > 1)
 343                                         dst_virt_ptr[PTR_PG(i - 2)]
 344                                                     [PTR_IDX(i - 2)] |=
 345                                                     PTU_PTE_NEXT_TO_LAST;
 346                         }
 347                 }
 348         }
 349 done:
 350         hwq->prod = 0;
 351         hwq->cons = 0;
 352         hwq->pdev = pdev;
 353         hwq->depth = hwq_attr->depth;
 354         hwq->max_elements = depth;
 355         hwq->element_size = stride;
 356         hwq->qe_ppg = pg_size / stride;
 357         /* For direct access to the elements */
 358         lvl = hwq->level;
 359         if (hwq_attr->sginfo->nopte && hwq->level)
 360                 lvl = hwq->level - 1;
 361         hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
 362         hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
 363         spin_lock_init(&hwq->lock);
 364
 365         return 0;
 366 fail:
 367         bnxt_qplib_free_hwq(res, hwq);
 368         return -ENOMEM;
 369 }
 370
 371 /* Context Tables */
 372 void bnxt_qplib_free_ctx(struct bnxt_qplib_res *res,
 373                          struct bnxt_qplib_ctx *ctx)
 374 {
 375         int i;
 376
 377         bnxt_qplib_free_hwq(res, &ctx->qpc_tbl);
 378         bnxt_qplib_free_hwq(res, &ctx->mrw_tbl);
 379         bnxt_qplib_free_hwq(res, &ctx->srqc_tbl);
 380         bnxt_qplib_free_hwq(res, &ctx->cq_tbl);
 381         bnxt_qplib_free_hwq(res, &ctx->tim_tbl);
 382         for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
 383                 bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.qtbl[i]);
 384         /* restore original pde level before destroy */
 385         ctx->tqm_ctx.pde.level = ctx->tqm_ctx.pde_level;
 386         bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.pde);
 387         bnxt_qplib_free_stats_ctx(res->pdev, &ctx->stats);
 388 }
 389
 390 static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
 391                                       struct bnxt_qplib_ctx *ctx)
 392 {
 393         struct bnxt_qplib_hwq_attr hwq_attr = {};
 394         struct bnxt_qplib_sg_info sginfo = {};
 395         struct bnxt_qplib_tqm_ctx *tqmctx;
 396         int rc = 0;
 397         int i;
 398
 399         tqmctx = &ctx->tqm_ctx;
 400
 401         sginfo.pgsize = PAGE_SIZE;
 402         sginfo.pgshft = PAGE_SHIFT;
 403         hwq_attr.sginfo = &sginfo;
 404         hwq_attr.res = res;
 405         hwq_attr.type = HWQ_TYPE_CTX;
 406         hwq_attr.depth = 512;
 407         hwq_attr.stride = sizeof(u64);
 408         /* Alloc pdl buffer */
 409         rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
 410         if (rc)
 411                 goto out;
 412         /* Save original pdl level */
 413         tqmctx->pde_level = tqmctx->pde.level;
 414
 415         hwq_attr.stride = 1;
 416         for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
 417                 if (!tqmctx->qcount[i])
 418                         continue;
 419                 hwq_attr.depth = ctx->qpc_count * tqmctx->qcount[i];
 420                 rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
 421                 if (rc)
 422                         goto out;
 423         }
 424 out:
 425         return rc;
 426 }
 427
 428 static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
 429 {
 430         struct bnxt_qplib_hwq *tbl;
 431         dma_addr_t *dma_ptr;
 432         __le64 **pbl_ptr, *ptr;
 433         int i, j, k;
 434         int fnz_idx = -1;
 435         int pg_count;
 436
 437         pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
 438
 439         for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
 440              i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
 441                 tbl = &ctx->qtbl[i];
 442                 if (!tbl->max_elements)
 443                         continue;
 444                 if (fnz_idx == -1)
 445                         fnz_idx = i; /* first non-zero index */
 446                 switch (tbl->level) {
 447                 case PBL_LVL_2:
 448                         pg_count = tbl->pbl[PBL_LVL_1].pg_count;
 449                         for (k = 0; k < pg_count; k++) {
 450                                 ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
 451                                 dma_ptr = &tbl->pbl[PBL_LVL_1].pg_map_arr[k];
 452                                 *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
 453                         }
 454                         break;
 455                 case PBL_LVL_1:
 456                 case PBL_LVL_0:
 457                 default:
 458                         ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
 459                         *ptr = cpu_to_le64(tbl->pbl[PBL_LVL_0].pg_map_arr[0] |
 460                                            PTU_PTE_VALID);
 461                         break;
 462                 }
 463         }
 464         if (fnz_idx == -1)
 465                 fnz_idx = 0;
 466         /* update pde level as per page table programming */
 467         ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
 468                           ctx->qtbl[fnz_idx].level + 1;
 469 }
 470
 471 static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
 472                                       struct bnxt_qplib_ctx *ctx)
 473 {
 474         int rc = 0;
 475
 476         rc = bnxt_qplib_alloc_tqm_rings(res, ctx);
 477         if (rc)
 478                 goto fail;
 479
 480         bnxt_qplib_map_tqm_pgtbl(&ctx->tqm_ctx);
 481 fail:
 482         return rc;
 483 }
 484
 485 /*
 486  * Routine: bnxt_qplib_alloc_ctx
 487  * Description:
 488  *     Context tables are memories which are used by the chip fw.
 489  *     The 6 tables defined are:
 490  *             QPC ctx - holds QP states
 491  *             MRW ctx - holds memory region and window
 492  *             SRQ ctx - holds shared RQ states
 493  *             CQ ctx - holds completion queue states
 494  *             TQM ctx - holds Tx Queue Manager context
 495  *             TIM ctx - holds timer context
 496  *     Depending on the size of the tbl requested, either a 1 Page Buffer List
 497  *     or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
 498  *     instead.
 499  *     Table might be employed as follows:
 500  *             For 0      < ctx size <= 1 PAGE, 0 level of ind is used
 501  *             For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
 502  *             For 512    < ctx size <= MAX, 2 levels of ind is used
 503  * Returns:
 504  *     0 if success, else -ERRORS
 505  */
 506 int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res,
 507                          struct bnxt_qplib_ctx *ctx,
 508                          bool virt_fn, bool is_p5)
 509 {
 510         struct bnxt_qplib_hwq_attr hwq_attr = {};
 511         struct bnxt_qplib_sg_info sginfo = {};
 512         int rc = 0;
 513
 514         if (virt_fn || is_p5)
 515                 goto stats_alloc;
 516
 517         /* QPC Tables */
 518         sginfo.pgsize = PAGE_SIZE;
 519         sginfo.pgshft = PAGE_SHIFT;
 520         hwq_attr.sginfo = &sginfo;
 521
 522         hwq_attr.res = res;
 523         hwq_attr.depth = ctx->qpc_count;
 524         hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
 525         hwq_attr.type = HWQ_TYPE_CTX;
 526         rc = bnxt_qplib_alloc_init_hwq(&ctx->qpc_tbl, &hwq_attr);
 527         if (rc)
 528                 goto fail;
 529
 530         /* MRW Tables */
 531         hwq_attr.depth = ctx->mrw_count;
 532         hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
 533         rc = bnxt_qplib_alloc_init_hwq(&ctx->mrw_tbl, &hwq_attr);
 534         if (rc)
 535                 goto fail;
 536
 537         /* SRQ Tables */
 538         hwq_attr.depth = ctx->srqc_count;
 539         hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
 540         rc = bnxt_qplib_alloc_init_hwq(&ctx->srqc_tbl, &hwq_attr);
 541         if (rc)
 542                 goto fail;
 543
 544         /* CQ Tables */
 545         hwq_attr.depth = ctx->cq_count;
 546         hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
 547         rc = bnxt_qplib_alloc_init_hwq(&ctx->cq_tbl, &hwq_attr);
 548         if (rc)
 549                 goto fail;
 550
 551         /* TQM Buffer */
 552         rc = bnxt_qplib_setup_tqm_rings(res, ctx);
 553         if (rc)
 554                 goto fail;
 555         /* TIM Buffer */
 556         ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
 557         hwq_attr.depth = ctx->qpc_count * 16;
 558         hwq_attr.stride = 1;
 559         rc = bnxt_qplib_alloc_init_hwq(&ctx->tim_tbl, &hwq_attr);
 560         if (rc)
 561                 goto fail;
 562 stats_alloc:
 563         /* Stats */
 564         rc = bnxt_qplib_alloc_stats_ctx(res->pdev, res->cctx, &ctx->stats);
 565         if (rc)
 566                 goto fail;
 567
 568         return 0;
 569
 570 fail:
 571         bnxt_qplib_free_ctx(res, ctx);
 572         return rc;
 573 }
 574
 575 static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
 576                                      struct bnxt_qplib_sgid_tbl *sgid_tbl)
 577 {
 578         kfree(sgid_tbl->tbl);
 579         kfree(sgid_tbl->hw_id);
 580         kfree(sgid_tbl->ctx);
 581         kfree(sgid_tbl->vlan);
 582         sgid_tbl->tbl = NULL;
 583         sgid_tbl->hw_id = NULL;
 584         sgid_tbl->ctx = NULL;
 585         sgid_tbl->vlan = NULL;
 586         sgid_tbl->max = 0;
 587         sgid_tbl->active = 0;
 588 }
 589
 590 static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
 591                                      struct bnxt_qplib_sgid_tbl *sgid_tbl,
 592                                      u16 max)
 593 {
 594         sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
 595         if (!sgid_tbl->tbl)
 596                 return -ENOMEM;
 597
 598         sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL);
 599         if (!sgid_tbl->hw_id)
 600                 goto out_free1;
 601
 602         sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
 603         if (!sgid_tbl->ctx)
 604                 goto out_free2;
 605
 606         sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
 607         if (!sgid_tbl->vlan)
 608                 goto out_free3;
 609
 610         sgid_tbl->max = max;
 611         return 0;
 612 out_free3:
 613         kfree(sgid_tbl->ctx);
 614         sgid_tbl->ctx = NULL;
 615 out_free2:
 616         kfree(sgid_tbl->hw_id);
 617         sgid_tbl->hw_id = NULL;
 618 out_free1:
 619         kfree(sgid_tbl->tbl);
 620         sgid_tbl->tbl = NULL;
 621         return -ENOMEM;
 622 };
 623
 624 static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
 625                                         struct bnxt_qplib_sgid_tbl *sgid_tbl)
 626 {
 627         int i;
 628
 629         for (i = 0; i < sgid_tbl->max; i++) {
 630                 if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
 631                            sizeof(bnxt_qplib_gid_zero)))
 632                         bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid,
 633                                             sgid_tbl->tbl[i].vlan_id, true);
 634         }
 635         memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
 636         memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
 637         memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
 638         sgid_tbl->active = 0;
 639 }
 640
 641 static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 642                                      struct net_device *netdev)
 643 {
 644         u32 i;
 645
 646         for (i = 0; i < sgid_tbl->max; i++)
 647                 sgid_tbl->tbl[i].vlan_id = 0xffff;
 648
 649         memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
 650 }
 651
 652 /* PDs */
 653 int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)
 654 {
 655         u32 bit_num;
 656
 657         bit_num = find_first_bit(pdt->tbl, pdt->max);
 658         if (bit_num == pdt->max)
 659                 return -ENOMEM;
 660
 661         /* Found unused PD */
 662         clear_bit(bit_num, pdt->tbl);
 663         pd->id = bit_num;
 664         return 0;
 665 }
 666
 667 int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
 668                           struct bnxt_qplib_pd_tbl *pdt,
 669                           struct bnxt_qplib_pd *pd)
 670 {
 671         if (test_and_set_bit(pd->id, pdt->tbl)) {
 672                 dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
 673                          pd->id);
 674                 return -EINVAL;
 675         }
 676         pd->id = 0;
 677         return 0;
 678 }
 679
 680 static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
 681 {
 682         kfree(pdt->tbl);
 683         pdt->tbl = NULL;
 684         pdt->max = 0;
 685 }
 686
 687 static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
 688                                    struct bnxt_qplib_pd_tbl *pdt,
 689                                    u32 max)
 690 {
 691         u32 bytes;
 692
 693         bytes = max >> 3;
 694         if (!bytes)
 695                 bytes = 1;
 696         pdt->tbl = kmalloc(bytes, GFP_KERNEL);
 697         if (!pdt->tbl)
 698                 return -ENOMEM;
 699
 700         pdt->max = max;
 701         memset((u8 *)pdt->tbl, 0xFF, bytes);
 702
 703         return 0;
 704 }
 705
 706 /* DPIs */
 707 int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,
 708                          struct bnxt_qplib_dpi     *dpi,
 709                          void                      *app)
 710 {
 711         u32 bit_num;
 712
 713         bit_num = find_first_bit(dpit->tbl, dpit->max);
 714         if (bit_num == dpit->max)
 715                 return -ENOMEM;
 716
 717         /* Found unused DPI */
 718         clear_bit(bit_num, dpit->tbl);
 719         dpit->app_tbl[bit_num] = app;
 720
 721         dpi->dpi = bit_num;
 722         dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE);
 723         dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE);
 724
 725         return 0;
 726 }
 727
 728 int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
 729                            struct bnxt_qplib_dpi_tbl *dpit,
 730                            struct bnxt_qplib_dpi     *dpi)
 731 {
 732         if (dpi->dpi >= dpit->max) {
 733                 dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d\n", dpi->dpi);
 734                 return -EINVAL;
 735         }
 736         if (test_and_set_bit(dpi->dpi, dpit->tbl)) {
 737                 dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d\n",
 738                          dpi->dpi);
 739                 return -EINVAL;
 740         }
 741         if (dpit->app_tbl)
 742                 dpit->app_tbl[dpi->dpi] = NULL;
 743         memset(dpi, 0, sizeof(*dpi));
 744
 745         return 0;
 746 }
 747
 748 static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res     *res,
 749                                     struct bnxt_qplib_dpi_tbl *dpit)
 750 {
 751         kfree(dpit->tbl);
 752         kfree(dpit->app_tbl);
 753         if (dpit->dbr_bar_reg_iomem)
 754                 pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
 755         memset(dpit, 0, sizeof(*dpit));
 756 }
 757
 758 static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res     *res,
 759                                     struct bnxt_qplib_dpi_tbl *dpit,
 760                                     u32                       dbr_offset)
 761 {
 762         u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION;
 763         resource_size_t bar_reg_base;
 764         u32 dbr_len, bytes;
 765
 766         if (dpit->dbr_bar_reg_iomem) {
 767                 dev_err(&res->pdev->dev, "DBR BAR region %d already mapped\n",
 768                         dbr_bar_reg);
 769                 return -EALREADY;
 770         }
 771
 772         bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg);
 773         if (!bar_reg_base) {
 774                 dev_err(&res->pdev->dev, "BAR region %d resc start failed\n",
 775                         dbr_bar_reg);
 776                 return -ENOMEM;
 777         }
 778
 779         dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset;
 780         if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) {
 781                 dev_err(&res->pdev->dev, "Invalid DBR length %d\n", dbr_len);
 782                 return -ENOMEM;
 783         }
 784
 785         dpit->dbr_bar_reg_iomem = ioremap(bar_reg_base + dbr_offset,
 786                                                   dbr_len);
 787         if (!dpit->dbr_bar_reg_iomem) {
 788                 dev_err(&res->pdev->dev,
 789                         "FP: DBR BAR region %d mapping failed\n", dbr_bar_reg);
 790                 return -ENOMEM;
 791         }
 792
 793         dpit->unmapped_dbr = bar_reg_base + dbr_offset;
 794         dpit->max = dbr_len / PAGE_SIZE;
 795
 796         dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
 797         if (!dpit->app_tbl)
 798                 goto unmap_io;
 799
 800         bytes = dpit->max >> 3;
 801         if (!bytes)
 802                 bytes = 1;
 803
 804         dpit->tbl = kmalloc(bytes, GFP_KERNEL);
 805         if (!dpit->tbl) {
 806                 kfree(dpit->app_tbl);
 807                 dpit->app_tbl = NULL;
 808                 goto unmap_io;
 809         }
 810
 811         memset((u8 *)dpit->tbl, 0xFF, bytes);
 812
 813         return 0;
 814
 815 unmap_io:
 816         pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
 817         dpit->dbr_bar_reg_iomem = NULL;
 818         return -ENOMEM;
 819 }
 820
 821 /* Stats */
 822 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
 823                                       struct bnxt_qplib_stats *stats)
 824 {
 825         if (stats->dma) {
 826                 dma_free_coherent(&pdev->dev, stats->size,
 827                                   stats->dma, stats->dma_map);
 828         }
 829         memset(stats, 0, sizeof(*stats));
 830         stats->fw_id = -1;
 831 }
 832
 833 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
 834                                       struct bnxt_qplib_chip_ctx *cctx,
 835                                       struct bnxt_qplib_stats *stats)
 836 {
 837         memset(stats, 0, sizeof(*stats));
 838         stats->fw_id = -1;
 839         stats->size = cctx->hw_stats_size;
 840         stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
 841                                         &stats->dma_map, GFP_KERNEL);
 842         if (!stats->dma) {
 843                 dev_err(&pdev->dev, "Stats DMA allocation failed\n");
 844                 return -ENOMEM;
 845         }
 846         return 0;
 847 }
 848
 849 void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
 850 {
 851         bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
 852 }
 853
 854 int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
 855 {
 856         bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
 857
 858         return 0;
 859 }
 860
 861 void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
 862 {
 863         bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
 864         bnxt_qplib_free_pd_tbl(&res->pd_tbl);
 865         bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
 866 }
 867
 868 int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
 869                          struct net_device *netdev,
 870                          struct bnxt_qplib_dev_attr *dev_attr)
 871 {
 872         int rc = 0;
 873
 874         res->pdev = pdev;
 875         res->netdev = netdev;
 876
 877         rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid);
 878         if (rc)
 879                 goto fail;
 880
 881         rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
 882         if (rc)
 883                 goto fail;
 884
 885         rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size);
 886         if (rc)
 887                 goto fail;
 888
 889         return 0;
 890 fail:
 891         bnxt_qplib_free_res(res);
 892         return rc;
 893 }
 894
 895 int bnxt_qplib_determine_atomics(struct pci_dev *dev)
 896 {
 897         int comp;
 898         u16 ctl2;
 899
 900         comp = pci_enable_atomic_ops_to_root(dev,
 901                                              PCI_EXP_DEVCAP2_ATOMIC_COMP32);
 902         if (comp)
 903                 return -EOPNOTSUPP;
 904         comp = pci_enable_atomic_ops_to_root(dev,
 905                                              PCI_EXP_DEVCAP2_ATOMIC_COMP64);
 906         if (comp)
 907                 return -EOPNOTSUPP;
 908         pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2);
 909         return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
 910 }