drivers/accel/qaic/qaic_data.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
   4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
   5
   6 #include <linux/bitfield.h>
   7 #include <linux/bits.h>
   8 #include <linux/completion.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-buf.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/kref.h>
  14 #include <linux/list.h>
  15 #include <linux/math64.h>
  16 #include <linux/mm.h>
  17 #include <linux/moduleparam.h>
  18 #include <linux/scatterlist.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/srcu.h>
  21 #include <linux/types.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/wait.h>
  24 #include <drm/drm_file.h>
  25 #include <drm/drm_gem.h>
  26 #include <drm/drm_prime.h>
  27 #include <drm/drm_print.h>
  28 #include <uapi/drm/qaic_accel.h>
  29
  30 #include "qaic.h"
  31
  32 #define SEM_VAL_MASK    GENMASK_ULL(11, 0)
  33 #define SEM_INDEX_MASK  GENMASK_ULL(4, 0)
  34 #define BULK_XFER       BIT(3)
  35 #define GEN_COMPLETION  BIT(4)
  36 #define INBOUND_XFER    1
  37 #define OUTBOUND_XFER   2
  38 #define REQHP_OFF       0x0 /* we read this */
  39 #define REQTP_OFF       0x4 /* we write this */
  40 #define RSPHP_OFF       0x8 /* we write this */
  41 #define RSPTP_OFF       0xc /* we read this */
  42
  43 #define ENCODE_SEM(val, index, sync, cmd, flags)                        \
  44                 ({                                                      \
  45                         FIELD_PREP(GENMASK(11, 0), (val)) |             \
  46                         FIELD_PREP(GENMASK(20, 16), (index)) |          \
  47                         FIELD_PREP(BIT(22), (sync)) |                   \
  48                         FIELD_PREP(GENMASK(26, 24), (cmd)) |            \
  49                         FIELD_PREP(GENMASK(30, 29), (flags)) |          \
  50                         FIELD_PREP(BIT(31), (cmd) ? 1 : 0);             \
  51                 })
  52 #define NUM_EVENTS      128
  53 #define NUM_DELAYS      10
  54
  55 static unsigned int wait_exec_default_timeout_ms = 5000; /* 5 sec default */
  56 module_param(wait_exec_default_timeout_ms, uint, 0600);
  57 MODULE_PARM_DESC(wait_exec_default_timeout_ms, "Default timeout for DRM_IOCTL_QAIC_WAIT_BO");
  58
  59 static unsigned int datapath_poll_interval_us = 100; /* 100 usec default */
  60 module_param(datapath_poll_interval_us, uint, 0600);
  61 MODULE_PARM_DESC(datapath_poll_interval_us,
  62                  "Amount of time to sleep between activity when datapath polling is enabled");
  63
  64 struct dbc_req {
  65         /*
  66          * A request ID is assigned to each memory handle going in DMA queue.
  67          * As a single memory handle can enqueue multiple elements in DMA queue
  68          * all of them will have the same request ID.
  69          */
  70         __le16  req_id;
  71         /* Future use */
  72         __u8    seq_id;
  73         /*
  74          * Special encoded variable
  75          * 7    0 - Do not force to generate MSI after DMA is completed
  76          *      1 - Force to generate MSI after DMA is completed
  77          * 6:5  Reserved
  78          * 4    1 - Generate completion element in the response queue
  79          *      0 - No Completion Code
  80          * 3    0 - DMA request is a Link list transfer
  81          *      1 - DMA request is a Bulk transfer
  82          * 2    Reserved
  83          * 1:0  00 - No DMA transfer involved
  84          *      01 - DMA transfer is part of inbound transfer
  85          *      10 - DMA transfer has outbound transfer
  86          *      11 - NA
  87          */
  88         __u8    cmd;
  89         __le32  resv;
  90         /* Source address for the transfer */
  91         __le64  src_addr;
  92         /* Destination address for the transfer */
  93         __le64  dest_addr;
  94         /* Length of transfer request */
  95         __le32  len;
  96         __le32  resv2;
  97         /* Doorbell address */
  98         __le64  db_addr;
  99         /*
 100          * Special encoded variable
 101          * 7    1 - Doorbell(db) write
 102          *      0 - No doorbell write
 103          * 6:2  Reserved
 104          * 1:0  00 - 32 bit access, db address must be aligned to 32bit-boundary
 105          *      01 - 16 bit access, db address must be aligned to 16bit-boundary
 106          *      10 - 8 bit access, db address must be aligned to 8bit-boundary
 107          *      11 - Reserved
 108          */
 109         __u8    db_len;
 110         __u8    resv3;
 111         __le16  resv4;
 112         /* 32 bit data written to doorbell address */
 113         __le32  db_data;
 114         /*
 115          * Special encoded variable
 116          * All the fields of sem_cmdX are passed from user and all are ORed
 117          * together to form sem_cmd.
 118          * 0:11         Semaphore value
 119          * 15:12        Reserved
 120          * 20:16        Semaphore index
 121          * 21           Reserved
 122          * 22           Semaphore Sync
 123          * 23           Reserved
 124          * 26:24        Semaphore command
 125          * 28:27        Reserved
 126          * 29           Semaphore DMA out bound sync fence
 127          * 30           Semaphore DMA in bound sync fence
 128          * 31           Enable semaphore command
 129          */
 130         __le32  sem_cmd0;
 131         __le32  sem_cmd1;
 132         __le32  sem_cmd2;
 133         __le32  sem_cmd3;
 134 } __packed;
 135
 136 struct dbc_rsp {
 137         /* Request ID of the memory handle whose DMA transaction is completed */
 138         __le16  req_id;
 139         /* Status of the DMA transaction. 0 : Success otherwise failure */
 140         __le16  status;
 141 } __packed;
 142
 143 inline int get_dbc_req_elem_size(void)
 144 {
 145         return sizeof(struct dbc_req);
 146 }
 147
 148 inline int get_dbc_rsp_elem_size(void)
 149 {
 150         return sizeof(struct dbc_rsp);
 151 }
 152
 153 static void free_slice(struct kref *kref)
 154 {
 155         struct bo_slice *slice = container_of(kref, struct bo_slice, ref_count);
 156
 157         list_del(&slice->slice);
 158         drm_gem_object_put(&slice->bo->base);
 159         sg_free_table(slice->sgt);
 160         kfree(slice->sgt);
 161         kfree(slice->reqs);
 162         kfree(slice);
 163 }
 164
 165 static int clone_range_of_sgt_for_slice(struct qaic_device *qdev, struct sg_table **sgt_out,
 166                                         struct sg_table *sgt_in, u64 size, u64 offset)
 167 {
 168         int total_len, len, nents, offf = 0, offl = 0;
 169         struct scatterlist *sg, *sgn, *sgf, *sgl;
 170         struct sg_table *sgt;
 171         int ret, j;
 172
 173         /* find out number of relevant nents needed for this mem */
 174         total_len = 0;
 175         sgf = NULL;
 176         sgl = NULL;
 177         nents = 0;
 178
 179         size = size ? size : PAGE_SIZE;
 180         for (sg = sgt_in->sgl; sg; sg = sg_next(sg)) {
 181                 len = sg_dma_len(sg);
 182
 183                 if (!len)
 184                         continue;
 185                 if (offset >= total_len && offset < total_len + len) {
 186                         sgf = sg;
 187                         offf = offset - total_len;
 188                 }
 189                 if (sgf)
 190                         nents++;
 191                 if (offset + size >= total_len &&
 192                     offset + size <= total_len + len) {
 193                         sgl = sg;
 194                         offl = offset + size - total_len;
 195                         break;
 196                 }
 197                 total_len += len;
 198         }
 199
 200         if (!sgf || !sgl) {
 201                 ret = -EINVAL;
 202                 goto out;
 203         }
 204
 205         sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
 206         if (!sgt) {
 207                 ret = -ENOMEM;
 208                 goto out;
 209         }
 210
 211         ret = sg_alloc_table(sgt, nents, GFP_KERNEL);
 212         if (ret)
 213                 goto free_sgt;
 214
 215         /* copy relevant sg node and fix page and length */
 216         sgn = sgf;
 217         for_each_sgtable_sg(sgt, sg, j) {
 218                 memcpy(sg, sgn, sizeof(*sg));
 219                 if (sgn == sgf) {
 220                         sg_dma_address(sg) += offf;
 221                         sg_dma_len(sg) -= offf;
 222                         sg_set_page(sg, sg_page(sgn), sg_dma_len(sg), offf);
 223                 } else {
 224                         offf = 0;
 225                 }
 226                 if (sgn == sgl) {
 227                         sg_dma_len(sg) = offl - offf;
 228                         sg_set_page(sg, sg_page(sgn), offl - offf, offf);
 229                         sg_mark_end(sg);
 230                         break;
 231                 }
 232                 sgn = sg_next(sgn);
 233         }
 234
 235         *sgt_out = sgt;
 236         return ret;
 237
 238 free_sgt:
 239         kfree(sgt);
 240 out:
 241         *sgt_out = NULL;
 242         return ret;
 243 }
 244
 245 static int encode_reqs(struct qaic_device *qdev, struct bo_slice *slice,
 246                        struct qaic_attach_slice_entry *req)
 247 {
 248         __le64 db_addr = cpu_to_le64(req->db_addr);
 249         __le32 db_data = cpu_to_le32(req->db_data);
 250         struct scatterlist *sg;
 251         __u8 cmd = BULK_XFER;
 252         int presync_sem;
 253         u64 dev_addr;
 254         __u8 db_len;
 255         int i;
 256
 257         if (!slice->no_xfer)
 258                 cmd |= (slice->dir == DMA_TO_DEVICE ? INBOUND_XFER : OUTBOUND_XFER);
 259
 260         if (req->db_len && !IS_ALIGNED(req->db_addr, req->db_len / 8))
 261                 return -EINVAL;
 262
 263         presync_sem = req->sem0.presync + req->sem1.presync + req->sem2.presync + req->sem3.presync;
 264         if (presync_sem > 1)
 265                 return -EINVAL;
 266
 267         presync_sem = req->sem0.presync << 0 | req->sem1.presync << 1 |
 268                       req->sem2.presync << 2 | req->sem3.presync << 3;
 269
 270         switch (req->db_len) {
 271         case 32:
 272                 db_len = BIT(7);
 273                 break;
 274         case 16:
 275                 db_len = BIT(7) | 1;
 276                 break;
 277         case 8:
 278                 db_len = BIT(7) | 2;
 279                 break;
 280         case 0:
 281                 db_len = 0; /* doorbell is not active for this command */
 282                 break;
 283         default:
 284                 return -EINVAL; /* should never hit this */
 285         }
 286
 287         /*
 288          * When we end up splitting up a single request (ie a buf slice) into
 289          * multiple DMA requests, we have to manage the sync data carefully.
 290          * There can only be one presync sem. That needs to be on every xfer
 291          * so that the DMA engine doesn't transfer data before the receiver is
 292          * ready. We only do the doorbell and postsync sems after the xfer.
 293          * To guarantee previous xfers for the request are complete, we use a
 294          * fence.
 295          */
 296         dev_addr = req->dev_addr;
 297         for_each_sgtable_sg(slice->sgt, sg, i) {
 298                 slice->reqs[i].cmd = cmd;
 299                 slice->reqs[i].src_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ?
 300                                                       sg_dma_address(sg) : dev_addr);
 301                 slice->reqs[i].dest_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ?
 302                                                        dev_addr : sg_dma_address(sg));
 303                 /*
 304                  * sg_dma_len(sg) returns size of a DMA segment, maximum DMA
 305                  * segment size is set to UINT_MAX by qaic and hence return
 306                  * values of sg_dma_len(sg) can never exceed u32 range. So,
 307                  * by down sizing we are not corrupting the value.
 308                  */
 309                 slice->reqs[i].len = cpu_to_le32((u32)sg_dma_len(sg));
 310                 switch (presync_sem) {
 311                 case BIT(0):
 312                         slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val,
 313                                                                          req->sem0.index,
 314                                                                          req->sem0.presync,
 315                                                                          req->sem0.cmd,
 316                                                                          req->sem0.flags));
 317                         break;
 318                 case BIT(1):
 319                         slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val,
 320                                                                          req->sem1.index,
 321                                                                          req->sem1.presync,
 322                                                                          req->sem1.cmd,
 323                                                                          req->sem1.flags));
 324                         break;
 325                 case BIT(2):
 326                         slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val,
 327                                                                          req->sem2.index,
 328                                                                          req->sem2.presync,
 329                                                                          req->sem2.cmd,
 330                                                                          req->sem2.flags));
 331                         break;
 332                 case BIT(3):
 333                         slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val,
 334                                                                          req->sem3.index,
 335                                                                          req->sem3.presync,
 336                                                                          req->sem3.cmd,
 337                                                                          req->sem3.flags));
 338                         break;
 339                 }
 340                 dev_addr += sg_dma_len(sg);
 341         }
 342         /* add post transfer stuff to last segment */
 343         i--;
 344         slice->reqs[i].cmd |= GEN_COMPLETION;
 345         slice->reqs[i].db_addr = db_addr;
 346         slice->reqs[i].db_len = db_len;
 347         slice->reqs[i].db_data = db_data;
 348         /*
 349          * Add a fence if we have more than one request going to the hardware
 350          * representing the entirety of the user request, and the user request
 351          * has no presync condition.
 352          * Fences are expensive, so we try to avoid them. We rely on the
 353          * hardware behavior to avoid needing one when there is a presync
 354          * condition. When a presync exists, all requests for that same
 355          * presync will be queued into a fifo. Thus, since we queue the
 356          * post xfer activity only on the last request we queue, the hardware
 357          * will ensure that the last queued request is processed last, thus
 358          * making sure the post xfer activity happens at the right time without
 359          * a fence.
 360          */
 361         if (i && !presync_sem)
 362                 req->sem0.flags |= (slice->dir == DMA_TO_DEVICE ?
 363                                     QAIC_SEM_INSYNCFENCE : QAIC_SEM_OUTSYNCFENCE);
 364         slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val, req->sem0.index,
 365                                                          req->sem0.presync, req->sem0.cmd,
 366                                                          req->sem0.flags));
 367         slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val, req->sem1.index,
 368                                                          req->sem1.presync, req->sem1.cmd,
 369                                                          req->sem1.flags));
 370         slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val, req->sem2.index,
 371                                                          req->sem2.presync, req->sem2.cmd,
 372                                                          req->sem2.flags));
 373         slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val, req->sem3.index,
 374                                                          req->sem3.presync, req->sem3.cmd,
 375                                                          req->sem3.flags));
 376
 377         return 0;
 378 }
 379
 380 static int qaic_map_one_slice(struct qaic_device *qdev, struct qaic_bo *bo,
 381                               struct qaic_attach_slice_entry *slice_ent)
 382 {
 383         struct sg_table *sgt = NULL;
 384         struct bo_slice *slice;
 385         int ret;
 386
 387         ret = clone_range_of_sgt_for_slice(qdev, &sgt, bo->sgt, slice_ent->size, slice_ent->offset);
 388         if (ret)
 389                 goto out;
 390
 391         slice = kmalloc(sizeof(*slice), GFP_KERNEL);
 392         if (!slice) {
 393                 ret = -ENOMEM;
 394                 goto free_sgt;
 395         }
 396
 397         slice->reqs = kcalloc(sgt->nents, sizeof(*slice->reqs), GFP_KERNEL);
 398         if (!slice->reqs) {
 399                 ret = -ENOMEM;
 400                 goto free_slice;
 401         }
 402
 403         slice->no_xfer = !slice_ent->size;
 404         slice->sgt = sgt;
 405         slice->nents = sgt->nents;
 406         slice->dir = bo->dir;
 407         slice->bo = bo;
 408         slice->size = slice_ent->size;
 409         slice->offset = slice_ent->offset;
 410
 411         ret = encode_reqs(qdev, slice, slice_ent);
 412         if (ret)
 413                 goto free_req;
 414
 415         bo->total_slice_nents += sgt->nents;
 416         kref_init(&slice->ref_count);
 417         drm_gem_object_get(&bo->base);
 418         list_add_tail(&slice->slice, &bo->slices);
 419
 420         return 0;
 421
 422 free_req:
 423         kfree(slice->reqs);
 424 free_slice:
 425         kfree(slice);
 426 free_sgt:
 427         sg_free_table(sgt);
 428         kfree(sgt);
 429 out:
 430         return ret;
 431 }
 432
 433 static int create_sgt(struct qaic_device *qdev, struct sg_table **sgt_out, u64 size)
 434 {
 435         struct scatterlist *sg;
 436         struct sg_table *sgt;
 437         struct page **pages;
 438         int *pages_order;
 439         int buf_extra;
 440         int max_order;
 441         int nr_pages;
 442         int ret = 0;
 443         int i, j, k;
 444         int order;
 445
 446         if (size) {
 447                 nr_pages = DIV_ROUND_UP(size, PAGE_SIZE);
 448                 /*
 449                  * calculate how much extra we are going to allocate, to remove
 450                  * later
 451                  */
 452                 buf_extra = (PAGE_SIZE - size % PAGE_SIZE) % PAGE_SIZE;
 453                 max_order = min(MAX_ORDER - 1, get_order(size));
 454         } else {
 455                 /* allocate a single page for book keeping */
 456                 nr_pages = 1;
 457                 buf_extra = 0;
 458                 max_order = 0;
 459         }
 460
 461         pages = kvmalloc_array(nr_pages, sizeof(*pages) + sizeof(*pages_order), GFP_KERNEL);
 462         if (!pages) {
 463                 ret = -ENOMEM;
 464                 goto out;
 465         }
 466         pages_order = (void *)pages + sizeof(*pages) * nr_pages;
 467
 468         /*
 469          * Allocate requested memory using alloc_pages. It is possible to allocate
 470          * the requested memory in multiple chunks by calling alloc_pages
 471          * multiple times. Use SG table to handle multiple allocated pages.
 472          */
 473         i = 0;
 474         while (nr_pages > 0) {
 475                 order = min(get_order(nr_pages * PAGE_SIZE), max_order);
 476                 while (1) {
 477                         pages[i] = alloc_pages(GFP_KERNEL | GFP_HIGHUSER |
 478                                                __GFP_NOWARN | __GFP_ZERO |
 479                                                (order ? __GFP_NORETRY : __GFP_RETRY_MAYFAIL),
 480                                                order);
 481                         if (pages[i])
 482                                 break;
 483                         if (!order--) {
 484                                 ret = -ENOMEM;
 485                                 goto free_partial_alloc;
 486                         }
 487                 }
 488
 489                 max_order = order;
 490                 pages_order[i] = order;
 491
 492                 nr_pages -= 1 << order;
 493                 if (nr_pages <= 0)
 494                         /* account for over allocation */
 495                         buf_extra += abs(nr_pages) * PAGE_SIZE;
 496                 i++;
 497         }
 498
 499         sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
 500         if (!sgt) {
 501                 ret = -ENOMEM;
 502                 goto free_partial_alloc;
 503         }
 504
 505         if (sg_alloc_table(sgt, i, GFP_KERNEL)) {
 506                 ret = -ENOMEM;
 507                 goto free_sgt;
 508         }
 509
 510         /* Populate the SG table with the allocated memory pages */
 511         sg = sgt->sgl;
 512         for (k = 0; k < i; k++, sg = sg_next(sg)) {
 513                 /* Last entry requires special handling */
 514                 if (k < i - 1) {
 515                         sg_set_page(sg, pages[k], PAGE_SIZE << pages_order[k], 0);
 516                 } else {
 517                         sg_set_page(sg, pages[k], (PAGE_SIZE << pages_order[k]) - buf_extra, 0);
 518                         sg_mark_end(sg);
 519                 }
 520         }
 521
 522         kvfree(pages);
 523         *sgt_out = sgt;
 524         return ret;
 525
 526 free_sgt:
 527         kfree(sgt);
 528 free_partial_alloc:
 529         for (j = 0; j < i; j++)
 530                 __free_pages(pages[j], pages_order[j]);
 531         kvfree(pages);
 532 out:
 533         *sgt_out = NULL;
 534         return ret;
 535 }
 536
 537 static bool invalid_sem(struct qaic_sem *sem)
 538 {
 539         if (sem->val & ~SEM_VAL_MASK || sem->index & ~SEM_INDEX_MASK ||
 540             !(sem->presync == 0 || sem->presync == 1) || sem->pad ||
 541             sem->flags & ~(QAIC_SEM_INSYNCFENCE | QAIC_SEM_OUTSYNCFENCE) ||
 542             sem->cmd > QAIC_SEM_WAIT_GT_0)
 543                 return true;
 544         return false;
 545 }
 546
 547 static int qaic_validate_req(struct qaic_device *qdev, struct qaic_attach_slice_entry *slice_ent,
 548                              u32 count, u64 total_size)
 549 {
 550         int i;
 551
 552         for (i = 0; i < count; i++) {
 553                 if (!(slice_ent[i].db_len == 32 || slice_ent[i].db_len == 16 ||
 554                       slice_ent[i].db_len == 8 || slice_ent[i].db_len == 0) ||
 555                       invalid_sem(&slice_ent[i].sem0) || invalid_sem(&slice_ent[i].sem1) ||
 556                       invalid_sem(&slice_ent[i].sem2) || invalid_sem(&slice_ent[i].sem3))
 557                         return -EINVAL;
 558
 559                 if (slice_ent[i].offset + slice_ent[i].size > total_size)
 560                         return -EINVAL;
 561         }
 562
 563         return 0;
 564 }
 565
 566 static void qaic_free_sgt(struct sg_table *sgt)
 567 {
 568         struct scatterlist *sg;
 569
 570         for (sg = sgt->sgl; sg; sg = sg_next(sg))
 571                 if (sg_page(sg))
 572                         __free_pages(sg_page(sg), get_order(sg->length));
 573         sg_free_table(sgt);
 574         kfree(sgt);
 575 }
 576
 577 static void qaic_gem_print_info(struct drm_printer *p, unsigned int indent,
 578                                 const struct drm_gem_object *obj)
 579 {
 580         struct qaic_bo *bo = to_qaic_bo(obj);
 581
 582         drm_printf_indent(p, indent, "user requested size=%llu\n", bo->size);
 583 }
 584
 585 static const struct vm_operations_struct drm_vm_ops = {
 586         .open = drm_gem_vm_open,
 587         .close = drm_gem_vm_close,
 588 };
 589
 590 static int qaic_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 591 {
 592         struct qaic_bo *bo = to_qaic_bo(obj);
 593         unsigned long offset = 0;
 594         struct scatterlist *sg;
 595         int ret = 0;
 596
 597         if (obj->import_attach)
 598                 return -EINVAL;
 599
 600         for (sg = bo->sgt->sgl; sg; sg = sg_next(sg)) {
 601                 if (sg_page(sg)) {
 602                         ret = remap_pfn_range(vma, vma->vm_start + offset, page_to_pfn(sg_page(sg)),
 603                                               sg->length, vma->vm_page_prot);
 604                         if (ret)
 605                                 goto out;
 606                         offset += sg->length;
 607                 }
 608         }
 609
 610 out:
 611         return ret;
 612 }
 613
 614 static void qaic_free_object(struct drm_gem_object *obj)
 615 {
 616         struct qaic_bo *bo = to_qaic_bo(obj);
 617
 618         if (obj->import_attach) {
 619                 /* DMABUF/PRIME Path */
 620                 drm_prime_gem_destroy(obj, NULL);
 621         } else {
 622                 /* Private buffer allocation path */
 623                 qaic_free_sgt(bo->sgt);
 624         }
 625
 626         drm_gem_object_release(obj);
 627         kfree(bo);
 628 }
 629
 630 static const struct drm_gem_object_funcs qaic_gem_funcs = {
 631         .free = qaic_free_object,
 632         .print_info = qaic_gem_print_info,
 633         .mmap = qaic_gem_object_mmap,
 634         .vm_ops = &drm_vm_ops,
 635 };
 636
 637 static struct qaic_bo *qaic_alloc_init_bo(void)
 638 {
 639         struct qaic_bo *bo;
 640
 641         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
 642         if (!bo)
 643                 return ERR_PTR(-ENOMEM);
 644
 645         INIT_LIST_HEAD(&bo->slices);
 646         init_completion(&bo->xfer_done);
 647         complete_all(&bo->xfer_done);
 648
 649         return bo;
 650 }
 651
 652 int qaic_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
 653 {
 654         struct qaic_create_bo *args = data;
 655         int usr_rcu_id, qdev_rcu_id;
 656         struct drm_gem_object *obj;
 657         struct qaic_device *qdev;
 658         struct qaic_user *usr;
 659         struct qaic_bo *bo;
 660         size_t size;
 661         int ret;
 662
 663         if (args->pad)
 664                 return -EINVAL;
 665
 666         size = PAGE_ALIGN(args->size);
 667         if (size == 0)
 668                 return -EINVAL;
 669
 670         usr = file_priv->driver_priv;
 671         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
 672         if (!usr->qddev) {
 673                 ret = -ENODEV;
 674                 goto unlock_usr_srcu;
 675         }
 676
 677         qdev = usr->qddev->qdev;
 678         qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
 679         if (qdev->in_reset) {
 680                 ret = -ENODEV;
 681                 goto unlock_dev_srcu;
 682         }
 683
 684         bo = qaic_alloc_init_bo();
 685         if (IS_ERR(bo)) {
 686                 ret = PTR_ERR(bo);
 687                 goto unlock_dev_srcu;
 688         }
 689         obj = &bo->base;
 690
 691         drm_gem_private_object_init(dev, obj, size);
 692
 693         obj->funcs = &qaic_gem_funcs;
 694         ret = create_sgt(qdev, &bo->sgt, size);
 695         if (ret)
 696                 goto free_bo;
 697
 698         bo->size = args->size;
 699
 700         ret = drm_gem_handle_create(file_priv, obj, &args->handle);
 701         if (ret)
 702                 goto free_sgt;
 703
 704         bo->handle = args->handle;
 705         drm_gem_object_put(obj);
 706         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
 707         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
 708
 709         return 0;
 710
 711 free_sgt:
 712         qaic_free_sgt(bo->sgt);
 713 free_bo:
 714         kfree(bo);
 715 unlock_dev_srcu:
 716         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
 717 unlock_usr_srcu:
 718         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
 719         return ret;
 720 }
 721
 722 int qaic_mmap_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
 723 {
 724         struct qaic_mmap_bo *args = data;
 725         int usr_rcu_id, qdev_rcu_id;
 726         struct drm_gem_object *obj;
 727         struct qaic_device *qdev;
 728         struct qaic_user *usr;
 729         int ret;
 730
 731         usr = file_priv->driver_priv;
 732         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
 733         if (!usr->qddev) {
 734                 ret = -ENODEV;
 735                 goto unlock_usr_srcu;
 736         }
 737
 738         qdev = usr->qddev->qdev;
 739         qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
 740         if (qdev->in_reset) {
 741                 ret = -ENODEV;
 742                 goto unlock_dev_srcu;
 743         }
 744
 745         obj = drm_gem_object_lookup(file_priv, args->handle);
 746         if (!obj) {
 747                 ret = -ENOENT;
 748                 goto unlock_dev_srcu;
 749         }
 750
 751         ret = drm_gem_create_mmap_offset(obj);
 752         if (ret == 0)
 753                 args->offset = drm_vma_node_offset_addr(&obj->vma_node);
 754
 755         drm_gem_object_put(obj);
 756
 757 unlock_dev_srcu:
 758         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
 759 unlock_usr_srcu:
 760         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
 761         return ret;
 762 }
 763
 764 struct drm_gem_object *qaic_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf)
 765 {
 766         struct dma_buf_attachment *attach;
 767         struct drm_gem_object *obj;
 768         struct qaic_bo *bo;
 769         int ret;
 770
 771         bo = qaic_alloc_init_bo();
 772         if (IS_ERR(bo)) {
 773                 ret = PTR_ERR(bo);
 774                 goto out;
 775         }
 776
 777         obj = &bo->base;
 778         get_dma_buf(dma_buf);
 779
 780         attach = dma_buf_attach(dma_buf, dev->dev);
 781         if (IS_ERR(attach)) {
 782                 ret = PTR_ERR(attach);
 783                 goto attach_fail;
 784         }
 785
 786         if (!attach->dmabuf->size) {
 787                 ret = -EINVAL;
 788                 goto size_align_fail;
 789         }
 790
 791         drm_gem_private_object_init(dev, obj, attach->dmabuf->size);
 792         /*
 793          * skipping dma_buf_map_attachment() as we do not know the direction
 794          * just yet. Once the direction is known in the subsequent IOCTL to
 795          * attach slicing, we can do it then.
 796          */
 797
 798         obj->funcs = &qaic_gem_funcs;
 799         obj->import_attach = attach;
 800         obj->resv = dma_buf->resv;
 801
 802         return obj;
 803
 804 size_align_fail:
 805         dma_buf_detach(dma_buf, attach);
 806 attach_fail:
 807         dma_buf_put(dma_buf);
 808         kfree(bo);
 809 out:
 810         return ERR_PTR(ret);
 811 }
 812
 813 static int qaic_prepare_import_bo(struct qaic_bo *bo, struct qaic_attach_slice_hdr *hdr)
 814 {
 815         struct drm_gem_object *obj = &bo->base;
 816         struct sg_table *sgt;
 817         int ret;
 818
 819         if (obj->import_attach->dmabuf->size < hdr->size)
 820                 return -EINVAL;
 821
 822         sgt = dma_buf_map_attachment(obj->import_attach, hdr->dir);
 823         if (IS_ERR(sgt)) {
 824                 ret = PTR_ERR(sgt);
 825                 return ret;
 826         }
 827
 828         bo->sgt = sgt;
 829         bo->size = hdr->size;
 830
 831         return 0;
 832 }
 833
 834 static int qaic_prepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo,
 835                                   struct qaic_attach_slice_hdr *hdr)
 836 {
 837         int ret;
 838
 839         if (bo->size != hdr->size)
 840                 return -EINVAL;
 841
 842         ret = dma_map_sgtable(&qdev->pdev->dev, bo->sgt, hdr->dir, 0);
 843         if (ret)
 844                 return -EFAULT;
 845
 846         return 0;
 847 }
 848
 849 static int qaic_prepare_bo(struct qaic_device *qdev, struct qaic_bo *bo,
 850                            struct qaic_attach_slice_hdr *hdr)
 851 {
 852         int ret;
 853
 854         if (bo->base.import_attach)
 855                 ret = qaic_prepare_import_bo(bo, hdr);
 856         else
 857                 ret = qaic_prepare_export_bo(qdev, bo, hdr);
 858
 859         if (ret == 0)
 860                 bo->dir = hdr->dir;
 861
 862         return ret;
 863 }
 864
 865 static void qaic_unprepare_import_bo(struct qaic_bo *bo)
 866 {
 867         dma_buf_unmap_attachment(bo->base.import_attach, bo->sgt, bo->dir);
 868         bo->sgt = NULL;
 869         bo->size = 0;
 870 }
 871
 872 static void qaic_unprepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo)
 873 {
 874         dma_unmap_sgtable(&qdev->pdev->dev, bo->sgt, bo->dir, 0);
 875 }
 876
 877 static void qaic_unprepare_bo(struct qaic_device *qdev, struct qaic_bo *bo)
 878 {
 879         if (bo->base.import_attach)
 880                 qaic_unprepare_import_bo(bo);
 881         else
 882                 qaic_unprepare_export_bo(qdev, bo);
 883
 884         bo->dir = 0;
 885 }
 886
 887 static void qaic_free_slices_bo(struct qaic_bo *bo)
 888 {
 889         struct bo_slice *slice, *temp;
 890
 891         list_for_each_entry_safe(slice, temp, &bo->slices, slice)
 892                 kref_put(&slice->ref_count, free_slice);
 893 }
 894
 895 static int qaic_attach_slicing_bo(struct qaic_device *qdev, struct qaic_bo *bo,
 896                                   struct qaic_attach_slice_hdr *hdr,
 897                                   struct qaic_attach_slice_entry *slice_ent)
 898 {
 899         int ret, i;
 900
 901         for (i = 0; i < hdr->count; i++) {
 902                 ret = qaic_map_one_slice(qdev, bo, &slice_ent[i]);
 903                 if (ret) {
 904                         qaic_free_slices_bo(bo);
 905                         return ret;
 906                 }
 907         }
 908
 909         if (bo->total_slice_nents > qdev->dbc[hdr->dbc_id].nelem) {
 910                 qaic_free_slices_bo(bo);
 911                 return -ENOSPC;
 912         }
 913
 914         bo->sliced = true;
 915         bo->nr_slice = hdr->count;
 916         list_add_tail(&bo->bo_list, &qdev->dbc[hdr->dbc_id].bo_lists);
 917
 918         return 0;
 919 }
 920
 921 int qaic_attach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
 922 {
 923         struct qaic_attach_slice_entry *slice_ent;
 924         struct qaic_attach_slice *args = data;
 925         int rcu_id, usr_rcu_id, qdev_rcu_id;
 926         struct dma_bridge_chan  *dbc;
 927         struct drm_gem_object *obj;
 928         struct qaic_device *qdev;
 929         unsigned long arg_size;
 930         struct qaic_user *usr;
 931         u8 __user *user_data;
 932         struct qaic_bo *bo;
 933         int ret;
 934
 935         if (args->hdr.count == 0)
 936                 return -EINVAL;
 937
 938         arg_size = args->hdr.count * sizeof(*slice_ent);
 939         if (arg_size / args->hdr.count != sizeof(*slice_ent))
 940                 return -EINVAL;
 941
 942         if (args->hdr.size == 0)
 943                 return -EINVAL;
 944
 945         if (!(args->hdr.dir == DMA_TO_DEVICE || args->hdr.dir == DMA_FROM_DEVICE))
 946                 return -EINVAL;
 947
 948         if (args->data == 0)
 949                 return -EINVAL;
 950
 951         usr = file_priv->driver_priv;
 952         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
 953         if (!usr->qddev) {
 954                 ret = -ENODEV;
 955                 goto unlock_usr_srcu;
 956         }
 957
 958         qdev = usr->qddev->qdev;
 959         qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
 960         if (qdev->in_reset) {
 961                 ret = -ENODEV;
 962                 goto unlock_dev_srcu;
 963         }
 964
 965         if (args->hdr.dbc_id >= qdev->num_dbc) {
 966                 ret = -EINVAL;
 967                 goto unlock_dev_srcu;
 968         }
 969
 970         user_data = u64_to_user_ptr(args->data);
 971
 972         slice_ent = kzalloc(arg_size, GFP_KERNEL);
 973         if (!slice_ent) {
 974                 ret = -EINVAL;
 975                 goto unlock_dev_srcu;
 976         }
 977
 978         ret = copy_from_user(slice_ent, user_data, arg_size);
 979         if (ret) {
 980                 ret = -EFAULT;
 981                 goto free_slice_ent;
 982         }
 983
 984         ret = qaic_validate_req(qdev, slice_ent, args->hdr.count, args->hdr.size);
 985         if (ret)
 986                 goto free_slice_ent;
 987
 988         obj = drm_gem_object_lookup(file_priv, args->hdr.handle);
 989         if (!obj) {
 990                 ret = -ENOENT;
 991                 goto free_slice_ent;
 992         }
 993
 994         bo = to_qaic_bo(obj);
 995
 996         if (bo->sliced) {
 997                 ret = -EINVAL;
 998                 goto put_bo;
 999         }
1000
1001         dbc = &qdev->dbc[args->hdr.dbc_id];
1002         rcu_id = srcu_read_lock(&dbc->ch_lock);
1003         if (dbc->usr != usr) {
1004                 ret = -EINVAL;
1005                 goto unlock_ch_srcu;
1006         }
1007
1008         ret = qaic_prepare_bo(qdev, bo, &args->hdr);
1009         if (ret)
1010                 goto unlock_ch_srcu;
1011
1012         ret = qaic_attach_slicing_bo(qdev, bo, &args->hdr, slice_ent);
1013         if (ret)
1014                 goto unprepare_bo;
1015
1016         if (args->hdr.dir == DMA_TO_DEVICE)
1017                 dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, args->hdr.dir);
1018
1019         bo->dbc = dbc;
1020         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1021         drm_gem_object_put(obj);
1022         kfree(slice_ent);
1023         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1024         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1025
1026         return 0;
1027
1028 unprepare_bo:
1029         qaic_unprepare_bo(qdev, bo);
1030 unlock_ch_srcu:
1031         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1032 put_bo:
1033         drm_gem_object_put(obj);
1034 free_slice_ent:
1035         kfree(slice_ent);
1036 unlock_dev_srcu:
1037         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1038 unlock_usr_srcu:
1039         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1040         return ret;
1041 }
1042
1043 static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id,
1044                                  u32 head, u32 *ptail)
1045 {
1046         struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
1047         struct dbc_req *reqs = slice->reqs;
1048         u32 tail = *ptail;
1049         u32 avail;
1050
1051         avail = head - tail;
1052         if (head <= tail)
1053                 avail += dbc->nelem;
1054
1055         --avail;
1056
1057         if (avail < slice->nents)
1058                 return -EAGAIN;
1059
1060         if (tail + slice->nents > dbc->nelem) {
1061                 avail = dbc->nelem - tail;
1062                 avail = min_t(u32, avail, slice->nents);
1063                 memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
1064                        sizeof(*reqs) * avail);
1065                 reqs += avail;
1066                 avail = slice->nents - avail;
1067                 if (avail)
1068                         memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail);
1069         } else {
1070                 memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
1071                        sizeof(*reqs) * slice->nents);
1072         }
1073
1074         *ptail = (tail + slice->nents) % dbc->nelem;
1075
1076         return 0;
1077 }
1078
1079 /*
1080  * Based on the value of resize we may only need to transmit first_n
1081  * entries and the last entry, with last_bytes to send from the last entry.
1082  * Note that first_n could be 0.
1083  */
1084 static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice,
1085                                          u64 resize, u32 dbc_id, u32 head, u32 *ptail)
1086 {
1087         struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
1088         struct dbc_req *reqs = slice->reqs;
1089         struct dbc_req *last_req;
1090         u32 tail = *ptail;
1091         u64 total_bytes;
1092         u64 last_bytes;
1093         u32 first_n;
1094         u32 avail;
1095         int ret;
1096         int i;
1097
1098         avail = head - tail;
1099         if (head <= tail)
1100                 avail += dbc->nelem;
1101
1102         --avail;
1103
1104         total_bytes = 0;
1105         for (i = 0; i < slice->nents; i++) {
1106                 total_bytes += le32_to_cpu(reqs[i].len);
1107                 if (total_bytes >= resize)
1108                         break;
1109         }
1110
1111         if (total_bytes < resize) {
1112                 /* User space should have used the full buffer path. */
1113                 ret = -EINVAL;
1114                 return ret;
1115         }
1116
1117         first_n = i;
1118         last_bytes = i ? resize + le32_to_cpu(reqs[i].len) - total_bytes : resize;
1119
1120         if (avail < (first_n + 1))
1121                 return -EAGAIN;
1122
1123         if (first_n) {
1124                 if (tail + first_n > dbc->nelem) {
1125                         avail = dbc->nelem - tail;
1126                         avail = min_t(u32, avail, first_n);
1127                         memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
1128                                sizeof(*reqs) * avail);
1129                         last_req = reqs + avail;
1130                         avail = first_n - avail;
1131                         if (avail)
1132                                 memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail);
1133                 } else {
1134                         memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
1135                                sizeof(*reqs) * first_n);
1136                 }
1137         }
1138
1139         /* Copy over the last entry. Here we need to adjust len to the left over
1140          * size, and set src and dst to the entry it is copied to.
1141          */
1142         last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size();
1143         memcpy(last_req, reqs + slice->nents - 1, sizeof(*reqs));
1144
1145         /*
1146          * last_bytes holds size of a DMA segment, maximum DMA segment size is
1147          * set to UINT_MAX by qaic and hence last_bytes can never exceed u32
1148          * range. So, by down sizing we are not corrupting the value.
1149          */
1150         last_req->len = cpu_to_le32((u32)last_bytes);
1151         last_req->src_addr = reqs[first_n].src_addr;
1152         last_req->dest_addr = reqs[first_n].dest_addr;
1153
1154         *ptail = (tail + first_n + 1) % dbc->nelem;
1155
1156         return 0;
1157 }
1158
1159 static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *file_priv,
1160                                   struct qaic_execute_entry *exec, unsigned int count,
1161                                   bool is_partial, struct dma_bridge_chan *dbc, u32 head,
1162                                   u32 *tail)
1163 {
1164         struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec;
1165         struct drm_gem_object *obj;
1166         struct bo_slice *slice;
1167         unsigned long flags;
1168         struct qaic_bo *bo;
1169         bool queued;
1170         int i, j;
1171         int ret;
1172
1173         for (i = 0; i < count; i++) {
1174                 /*
1175                  * ref count will be decremented when the transfer of this
1176                  * buffer is complete. It is inside dbc_irq_threaded_fn().
1177                  */
1178                 obj = drm_gem_object_lookup(file_priv,
1179                                             is_partial ? pexec[i].handle : exec[i].handle);
1180                 if (!obj) {
1181                         ret = -ENOENT;
1182                         goto failed_to_send_bo;
1183                 }
1184
1185                 bo = to_qaic_bo(obj);
1186
1187                 if (!bo->sliced) {
1188                         ret = -EINVAL;
1189                         goto failed_to_send_bo;
1190                 }
1191
1192                 if (is_partial && pexec[i].resize > bo->size) {
1193                         ret = -EINVAL;
1194                         goto failed_to_send_bo;
1195                 }
1196
1197                 spin_lock_irqsave(&dbc->xfer_lock, flags);
1198                 queued = bo->queued;
1199                 bo->queued = true;
1200                 if (queued) {
1201                         spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1202                         ret = -EINVAL;
1203                         goto failed_to_send_bo;
1204                 }
1205
1206                 bo->req_id = dbc->next_req_id++;
1207
1208                 list_for_each_entry(slice, &bo->slices, slice) {
1209                         /*
1210                          * If this slice does not fall under the given
1211                          * resize then skip this slice and continue the loop
1212                          */
1213                         if (is_partial && pexec[i].resize && pexec[i].resize <= slice->offset)
1214                                 continue;
1215
1216                         for (j = 0; j < slice->nents; j++)
1217                                 slice->reqs[j].req_id = cpu_to_le16(bo->req_id);
1218
1219                         /*
1220                          * If it is a partial execute ioctl call then check if
1221                          * resize has cut this slice short then do a partial copy
1222                          * else do complete copy
1223                          */
1224                         if (is_partial && pexec[i].resize &&
1225                             pexec[i].resize < slice->offset + slice->size)
1226                                 ret = copy_partial_exec_reqs(qdev, slice,
1227                                                              pexec[i].resize - slice->offset,
1228                                                              dbc->id, head, tail);
1229                         else
1230                                 ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail);
1231                         if (ret) {
1232                                 bo->queued = false;
1233                                 spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1234                                 goto failed_to_send_bo;
1235                         }
1236                 }
1237                 reinit_completion(&bo->xfer_done);
1238                 list_add_tail(&bo->xfer_list, &dbc->xfer_list);
1239                 spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1240                 dma_sync_sgtable_for_device(&qdev->pdev->dev, bo->sgt, bo->dir);
1241         }
1242
1243         return 0;
1244
1245 failed_to_send_bo:
1246         if (likely(obj))
1247                 drm_gem_object_put(obj);
1248         for (j = 0; j < i; j++) {
1249                 spin_lock_irqsave(&dbc->xfer_lock, flags);
1250                 bo = list_last_entry(&dbc->xfer_list, struct qaic_bo, xfer_list);
1251                 obj = &bo->base;
1252                 bo->queued = false;
1253                 list_del(&bo->xfer_list);
1254                 spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1255                 dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);
1256                 drm_gem_object_put(obj);
1257         }
1258         return ret;
1259 }
1260
1261 static void update_profiling_data(struct drm_file *file_priv,
1262                                   struct qaic_execute_entry *exec, unsigned int count,
1263                                   bool is_partial, u64 received_ts, u64 submit_ts, u32 queue_level)
1264 {
1265         struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec;
1266         struct drm_gem_object *obj;
1267         struct qaic_bo *bo;
1268         int i;
1269
1270         for (i = 0; i < count; i++) {
1271                 /*
1272                  * Since we already committed the BO to hardware, the only way
1273                  * this should fail is a pending signal. We can't cancel the
1274                  * submit to hardware, so we have to just skip the profiling
1275                  * data. In case the signal is not fatal to the process, we
1276                  * return success so that the user doesn't try to resubmit.
1277                  */
1278                 obj = drm_gem_object_lookup(file_priv,
1279                                             is_partial ? pexec[i].handle : exec[i].handle);
1280                 if (!obj)
1281                         break;
1282                 bo = to_qaic_bo(obj);
1283                 bo->perf_stats.req_received_ts = received_ts;
1284                 bo->perf_stats.req_submit_ts = submit_ts;
1285                 bo->perf_stats.queue_level_before = queue_level;
1286                 queue_level += bo->total_slice_nents;
1287                 drm_gem_object_put(obj);
1288         }
1289 }
1290
1291 static int __qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv,
1292                                    bool is_partial)
1293 {
1294         struct qaic_execute *args = data;
1295         struct qaic_execute_entry *exec;
1296         struct dma_bridge_chan *dbc;
1297         int usr_rcu_id, qdev_rcu_id;
1298         struct qaic_device *qdev;
1299         struct qaic_user *usr;
1300         u8 __user *user_data;
1301         unsigned long n;
1302         u64 received_ts;
1303         u32 queue_level;
1304         u64 submit_ts;
1305         int rcu_id;
1306         u32 head;
1307         u32 tail;
1308         u64 size;
1309         int ret;
1310
1311         received_ts = ktime_get_ns();
1312
1313         size = is_partial ? sizeof(struct qaic_partial_execute_entry) : sizeof(*exec);
1314         n = (unsigned long)size * args->hdr.count;
1315         if (args->hdr.count == 0 || n / args->hdr.count != size)
1316                 return -EINVAL;
1317
1318         user_data = u64_to_user_ptr(args->data);
1319
1320         exec = kcalloc(args->hdr.count, size, GFP_KERNEL);
1321         if (!exec)
1322                 return -ENOMEM;
1323
1324         if (copy_from_user(exec, user_data, n)) {
1325                 ret = -EFAULT;
1326                 goto free_exec;
1327         }
1328
1329         usr = file_priv->driver_priv;
1330         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
1331         if (!usr->qddev) {
1332                 ret = -ENODEV;
1333                 goto unlock_usr_srcu;
1334         }
1335
1336         qdev = usr->qddev->qdev;
1337         qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
1338         if (qdev->in_reset) {
1339                 ret = -ENODEV;
1340                 goto unlock_dev_srcu;
1341         }
1342
1343         if (args->hdr.dbc_id >= qdev->num_dbc) {
1344                 ret = -EINVAL;
1345                 goto unlock_dev_srcu;
1346         }
1347
1348         dbc = &qdev->dbc[args->hdr.dbc_id];
1349
1350         rcu_id = srcu_read_lock(&dbc->ch_lock);
1351         if (!dbc->usr || dbc->usr->handle != usr->handle) {
1352                 ret = -EPERM;
1353                 goto release_ch_rcu;
1354         }
1355
1356         head = readl(dbc->dbc_base + REQHP_OFF);
1357         tail = readl(dbc->dbc_base + REQTP_OFF);
1358
1359         if (head == U32_MAX || tail == U32_MAX) {
1360                 /* PCI link error */
1361                 ret = -ENODEV;
1362                 goto release_ch_rcu;
1363         }
1364
1365         queue_level = head <= tail ? tail - head : dbc->nelem - (head - tail);
1366
1367         ret = send_bo_list_to_device(qdev, file_priv, exec, args->hdr.count, is_partial, dbc,
1368                                      head, &tail);
1369         if (ret)
1370                 goto release_ch_rcu;
1371
1372         /* Finalize commit to hardware */
1373         submit_ts = ktime_get_ns();
1374         writel(tail, dbc->dbc_base + REQTP_OFF);
1375
1376         update_profiling_data(file_priv, exec, args->hdr.count, is_partial, received_ts,
1377                               submit_ts, queue_level);
1378
1379         if (datapath_polling)
1380                 schedule_work(&dbc->poll_work);
1381
1382 release_ch_rcu:
1383         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1384 unlock_dev_srcu:
1385         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1386 unlock_usr_srcu:
1387         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1388 free_exec:
1389         kfree(exec);
1390         return ret;
1391 }
1392
1393 int qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1394 {
1395         return __qaic_execute_bo_ioctl(dev, data, file_priv, false);
1396 }
1397
1398 int qaic_partial_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1399 {
1400         return __qaic_execute_bo_ioctl(dev, data, file_priv, true);
1401 }
1402
1403 /*
1404  * Our interrupt handling is a bit more complicated than a simple ideal, but
1405  * sadly necessary.
1406  *
1407  * Each dbc has a completion queue. Entries in the queue correspond to DMA
1408  * requests which the device has processed. The hardware already has a built
1409  * in irq mitigation. When the device puts an entry into the queue, it will
1410  * only trigger an interrupt if the queue was empty. Therefore, when adding
1411  * the Nth event to a non-empty queue, the hardware doesn't trigger an
1412  * interrupt. This means the host doesn't get additional interrupts signaling
1413  * the same thing - the queue has something to process.
1414  * This behavior can be overridden in the DMA request.
1415  * This means that when the host receives an interrupt, it is required to
1416  * drain the queue.
1417  *
1418  * This behavior is what NAPI attempts to accomplish, although we can't use
1419  * NAPI as we don't have a netdev. We use threaded irqs instead.
1420  *
1421  * However, there is a situation where the host drains the queue fast enough
1422  * that every event causes an interrupt. Typically this is not a problem as
1423  * the rate of events would be low. However, that is not the case with
1424  * lprnet for example. On an Intel Xeon D-2191 where we run 8 instances of
1425  * lprnet, the host receives roughly 80k interrupts per second from the device
1426  * (per /proc/interrupts). While NAPI documentation indicates the host should
1427  * just chug along, sadly that behavior causes instability in some hosts.
1428  *
1429  * Therefore, we implement an interrupt disable scheme similar to NAPI. The
1430  * key difference is that we will delay after draining the queue for a small
1431  * time to allow additional events to come in via polling. Using the above
1432  * lprnet workload, this reduces the number of interrupts processed from
1433  * ~80k/sec to about 64 in 5 minutes and appears to solve the system
1434  * instability.
1435  */
1436 irqreturn_t dbc_irq_handler(int irq, void *data)
1437 {
1438         struct dma_bridge_chan *dbc = data;
1439         int rcu_id;
1440         u32 head;
1441         u32 tail;
1442
1443         rcu_id = srcu_read_lock(&dbc->ch_lock);
1444
1445         if (!dbc->usr) {
1446                 srcu_read_unlock(&dbc->ch_lock, rcu_id);
1447                 return IRQ_HANDLED;
1448         }
1449
1450         head = readl(dbc->dbc_base + RSPHP_OFF);
1451         if (head == U32_MAX) { /* PCI link error */
1452                 srcu_read_unlock(&dbc->ch_lock, rcu_id);
1453                 return IRQ_NONE;
1454         }
1455
1456         tail = readl(dbc->dbc_base + RSPTP_OFF);
1457         if (tail == U32_MAX) { /* PCI link error */
1458                 srcu_read_unlock(&dbc->ch_lock, rcu_id);
1459                 return IRQ_NONE;
1460         }
1461
1462         if (head == tail) { /* queue empty */
1463                 srcu_read_unlock(&dbc->ch_lock, rcu_id);
1464                 return IRQ_NONE;
1465         }
1466
1467         disable_irq_nosync(irq);
1468         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1469         return IRQ_WAKE_THREAD;
1470 }
1471
1472 void irq_polling_work(struct work_struct *work)
1473 {
1474         struct dma_bridge_chan *dbc = container_of(work, struct dma_bridge_chan,  poll_work);
1475         unsigned long flags;
1476         int rcu_id;
1477         u32 head;
1478         u32 tail;
1479
1480         rcu_id = srcu_read_lock(&dbc->ch_lock);
1481
1482         while (1) {
1483                 if (dbc->qdev->in_reset) {
1484                         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1485                         return;
1486                 }
1487                 if (!dbc->usr) {
1488                         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1489                         return;
1490                 }
1491                 spin_lock_irqsave(&dbc->xfer_lock, flags);
1492                 if (list_empty(&dbc->xfer_list)) {
1493                         spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1494                         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1495                         return;
1496                 }
1497                 spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1498
1499                 head = readl(dbc->dbc_base + RSPHP_OFF);
1500                 if (head == U32_MAX) { /* PCI link error */
1501                         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1502                         return;
1503                 }
1504
1505                 tail = readl(dbc->dbc_base + RSPTP_OFF);
1506                 if (tail == U32_MAX) { /* PCI link error */
1507                         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1508                         return;
1509                 }
1510
1511                 if (head != tail) {
1512                         irq_wake_thread(dbc->irq, dbc);
1513                         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1514                         return;
1515                 }
1516
1517                 cond_resched();
1518                 usleep_range(datapath_poll_interval_us, 2 * datapath_poll_interval_us);
1519         }
1520 }
1521
1522 irqreturn_t dbc_irq_threaded_fn(int irq, void *data)
1523 {
1524         struct dma_bridge_chan *dbc = data;
1525         int event_count = NUM_EVENTS;
1526         int delay_count = NUM_DELAYS;
1527         struct qaic_device *qdev;
1528         struct qaic_bo *bo, *i;
1529         struct dbc_rsp *rsp;
1530         unsigned long flags;
1531         int rcu_id;
1532         u16 status;
1533         u16 req_id;
1534         u32 head;
1535         u32 tail;
1536
1537         rcu_id = srcu_read_lock(&dbc->ch_lock);
1538
1539         head = readl(dbc->dbc_base + RSPHP_OFF);
1540         if (head == U32_MAX) /* PCI link error */
1541                 goto error_out;
1542
1543         qdev = dbc->qdev;
1544 read_fifo:
1545
1546         if (!event_count) {
1547                 event_count = NUM_EVENTS;
1548                 cond_resched();
1549         }
1550
1551         /*
1552          * if this channel isn't assigned or gets unassigned during processing
1553          * we have nothing further to do
1554          */
1555         if (!dbc->usr)
1556                 goto error_out;
1557
1558         tail = readl(dbc->dbc_base + RSPTP_OFF);
1559         if (tail == U32_MAX) /* PCI link error */
1560                 goto error_out;
1561
1562         if (head == tail) { /* queue empty */
1563                 if (delay_count) {
1564                         --delay_count;
1565                         usleep_range(100, 200);
1566                         goto read_fifo; /* check for a new event */
1567                 }
1568                 goto normal_out;
1569         }
1570
1571         delay_count = NUM_DELAYS;
1572         while (head != tail) {
1573                 if (!event_count)
1574                         break;
1575                 --event_count;
1576                 rsp = dbc->rsp_q_base + head * sizeof(*rsp);
1577                 req_id = le16_to_cpu(rsp->req_id);
1578                 status = le16_to_cpu(rsp->status);
1579                 if (status)
1580                         pci_dbg(qdev->pdev, "req_id %d failed with status %d\n", req_id, status);
1581                 spin_lock_irqsave(&dbc->xfer_lock, flags);
1582                 /*
1583                  * A BO can receive multiple interrupts, since a BO can be
1584                  * divided into multiple slices and a buffer receives as many
1585                  * interrupts as slices. So until it receives interrupts for
1586                  * all the slices we cannot mark that buffer complete.
1587                  */
1588                 list_for_each_entry_safe(bo, i, &dbc->xfer_list, xfer_list) {
1589                         if (bo->req_id == req_id)
1590                                 bo->nr_slice_xfer_done++;
1591                         else
1592                                 continue;
1593
1594                         if (bo->nr_slice_xfer_done < bo->nr_slice)
1595                                 break;
1596
1597                         /*
1598                          * At this point we have received all the interrupts for
1599                          * BO, which means BO execution is complete.
1600                          */
1601                         dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);
1602                         bo->nr_slice_xfer_done = 0;
1603                         bo->queued = false;
1604                         list_del(&bo->xfer_list);
1605                         bo->perf_stats.req_processed_ts = ktime_get_ns();
1606                         complete_all(&bo->xfer_done);
1607                         drm_gem_object_put(&bo->base);
1608                         break;
1609                 }
1610                 spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1611                 head = (head + 1) % dbc->nelem;
1612         }
1613
1614         /*
1615          * Update the head pointer of response queue and let the device know
1616          * that we have consumed elements from the queue.
1617          */
1618         writel(head, dbc->dbc_base + RSPHP_OFF);
1619
1620         /* elements might have been put in the queue while we were processing */
1621         goto read_fifo;
1622
1623 normal_out:
1624         if (likely(!datapath_polling))
1625                 enable_irq(irq);
1626         else
1627                 schedule_work(&dbc->poll_work);
1628         /* checking the fifo and enabling irqs is a race, missed event check */
1629         tail = readl(dbc->dbc_base + RSPTP_OFF);
1630         if (tail != U32_MAX && head != tail) {
1631                 if (likely(!datapath_polling))
1632                         disable_irq_nosync(irq);
1633                 goto read_fifo;
1634         }
1635         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1636         return IRQ_HANDLED;
1637
1638 error_out:
1639         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1640         if (likely(!datapath_polling))
1641                 enable_irq(irq);
1642         else
1643                 schedule_work(&dbc->poll_work);
1644
1645         return IRQ_HANDLED;
1646 }
1647
1648 int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1649 {
1650         struct qaic_wait *args = data;
1651         int usr_rcu_id, qdev_rcu_id;
1652         struct dma_bridge_chan *dbc;
1653         struct drm_gem_object *obj;
1654         struct qaic_device *qdev;
1655         unsigned long timeout;
1656         struct qaic_user *usr;
1657         struct qaic_bo *bo;
1658         int rcu_id;
1659         int ret;
1660
1661         if (args->pad != 0)
1662                 return -EINVAL;
1663
1664         usr = file_priv->driver_priv;
1665         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
1666         if (!usr->qddev) {
1667                 ret = -ENODEV;
1668                 goto unlock_usr_srcu;
1669         }
1670
1671         qdev = usr->qddev->qdev;
1672         qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
1673         if (qdev->in_reset) {
1674                 ret = -ENODEV;
1675                 goto unlock_dev_srcu;
1676         }
1677
1678         if (args->dbc_id >= qdev->num_dbc) {
1679                 ret = -EINVAL;
1680                 goto unlock_dev_srcu;
1681         }
1682
1683         dbc = &qdev->dbc[args->dbc_id];
1684
1685         rcu_id = srcu_read_lock(&dbc->ch_lock);
1686         if (dbc->usr != usr) {
1687                 ret = -EPERM;
1688                 goto unlock_ch_srcu;
1689         }
1690
1691         obj = drm_gem_object_lookup(file_priv, args->handle);
1692         if (!obj) {
1693                 ret = -ENOENT;
1694                 goto unlock_ch_srcu;
1695         }
1696
1697         bo = to_qaic_bo(obj);
1698         timeout = args->timeout ? args->timeout : wait_exec_default_timeout_ms;
1699         timeout = msecs_to_jiffies(timeout);
1700         ret = wait_for_completion_interruptible_timeout(&bo->xfer_done, timeout);
1701         if (!ret) {
1702                 ret = -ETIMEDOUT;
1703                 goto put_obj;
1704         }
1705         if (ret > 0)
1706                 ret = 0;
1707
1708         if (!dbc->usr)
1709                 ret = -EPERM;
1710
1711 put_obj:
1712         drm_gem_object_put(obj);
1713 unlock_ch_srcu:
1714         srcu_read_unlock(&dbc->ch_lock, rcu_id);
1715 unlock_dev_srcu:
1716         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1717 unlock_usr_srcu:
1718         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1719         return ret;
1720 }
1721
1722 int qaic_perf_stats_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1723 {
1724         struct qaic_perf_stats_entry *ent = NULL;
1725         struct qaic_perf_stats *args = data;
1726         int usr_rcu_id, qdev_rcu_id;
1727         struct drm_gem_object *obj;
1728         struct qaic_device *qdev;
1729         struct qaic_user *usr;
1730         struct qaic_bo *bo;
1731         int ret, i;
1732
1733         usr = file_priv->driver_priv;
1734         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
1735         if (!usr->qddev) {
1736                 ret = -ENODEV;
1737                 goto unlock_usr_srcu;
1738         }
1739
1740         qdev = usr->qddev->qdev;
1741         qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
1742         if (qdev->in_reset) {
1743                 ret = -ENODEV;
1744                 goto unlock_dev_srcu;
1745         }
1746
1747         if (args->hdr.dbc_id >= qdev->num_dbc) {
1748                 ret = -EINVAL;
1749                 goto unlock_dev_srcu;
1750         }
1751
1752         ent = kcalloc(args->hdr.count, sizeof(*ent), GFP_KERNEL);
1753         if (!ent) {
1754                 ret = -EINVAL;
1755                 goto unlock_dev_srcu;
1756         }
1757
1758         ret = copy_from_user(ent, u64_to_user_ptr(args->data), args->hdr.count * sizeof(*ent));
1759         if (ret) {
1760                 ret = -EFAULT;
1761                 goto free_ent;
1762         }
1763
1764         for (i = 0; i < args->hdr.count; i++) {
1765                 obj = drm_gem_object_lookup(file_priv, ent[i].handle);
1766                 if (!obj) {
1767                         ret = -ENOENT;
1768                         goto free_ent;
1769                 }
1770                 bo = to_qaic_bo(obj);
1771                 /*
1772                  * perf stats ioctl is called before wait ioctl is complete then
1773                  * the latency information is invalid.
1774                  */
1775                 if (bo->perf_stats.req_processed_ts < bo->perf_stats.req_submit_ts) {
1776                         ent[i].device_latency_us = 0;
1777                 } else {
1778                         ent[i].device_latency_us = div_u64((bo->perf_stats.req_processed_ts -
1779                                                             bo->perf_stats.req_submit_ts), 1000);
1780                 }
1781                 ent[i].submit_latency_us = div_u64((bo->perf_stats.req_submit_ts -
1782                                                     bo->perf_stats.req_received_ts), 1000);
1783                 ent[i].queue_level_before = bo->perf_stats.queue_level_before;
1784                 ent[i].num_queue_element = bo->total_slice_nents;
1785                 drm_gem_object_put(obj);
1786         }
1787
1788         if (copy_to_user(u64_to_user_ptr(args->data), ent, args->hdr.count * sizeof(*ent)))
1789                 ret = -EFAULT;
1790
1791 free_ent:
1792         kfree(ent);
1793 unlock_dev_srcu:
1794         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1795 unlock_usr_srcu:
1796         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1797         return ret;
1798 }
1799
1800 static void empty_xfer_list(struct qaic_device *qdev, struct dma_bridge_chan *dbc)
1801 {
1802         unsigned long flags;
1803         struct qaic_bo *bo;
1804
1805         spin_lock_irqsave(&dbc->xfer_lock, flags);
1806         while (!list_empty(&dbc->xfer_list)) {
1807                 bo = list_first_entry(&dbc->xfer_list, typeof(*bo), xfer_list);
1808                 bo->queued = false;
1809                 list_del(&bo->xfer_list);
1810                 spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1811                 dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);
1812                 complete_all(&bo->xfer_done);
1813                 drm_gem_object_put(&bo->base);
1814                 spin_lock_irqsave(&dbc->xfer_lock, flags);
1815         }
1816         spin_unlock_irqrestore(&dbc->xfer_lock, flags);
1817 }
1818
1819 int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr)
1820 {
1821         if (!qdev->dbc[dbc_id].usr || qdev->dbc[dbc_id].usr->handle != usr->handle)
1822                 return -EPERM;
1823
1824         qdev->dbc[dbc_id].usr = NULL;
1825         synchronize_srcu(&qdev->dbc[dbc_id].ch_lock);
1826         return 0;
1827 }
1828
1829 /**
1830  * enable_dbc - Enable the DBC. DBCs are disabled by removing the context of
1831  * user. Add user context back to DBC to enable it. This function trusts the
1832  * DBC ID passed and expects the DBC to be disabled.
1833  * @qdev: Qranium device handle
1834  * @dbc_id: ID of the DBC
1835  * @usr: User context
1836  */
1837 void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr)
1838 {
1839         qdev->dbc[dbc_id].usr = usr;
1840 }
1841
1842 void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id)
1843 {
1844         struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
1845
1846         dbc->usr = NULL;
1847         empty_xfer_list(qdev, dbc);
1848         synchronize_srcu(&dbc->ch_lock);
1849         /*
1850          * Threads holding channel lock, may add more elements in the xfer_list.
1851          * Flush out these elements from xfer_list.
1852          */
1853         empty_xfer_list(qdev, dbc);
1854 }
1855
1856 void release_dbc(struct qaic_device *qdev, u32 dbc_id)
1857 {
1858         struct bo_slice *slice, *slice_temp;
1859         struct qaic_bo *bo, *bo_temp;
1860         struct dma_bridge_chan *dbc;
1861
1862         dbc = &qdev->dbc[dbc_id];
1863         if (!dbc->in_use)
1864                 return;
1865
1866         wakeup_dbc(qdev, dbc_id);
1867
1868         dma_free_coherent(&qdev->pdev->dev, dbc->total_size, dbc->req_q_base, dbc->dma_addr);
1869         dbc->total_size = 0;
1870         dbc->req_q_base = NULL;
1871         dbc->dma_addr = 0;
1872         dbc->nelem = 0;
1873         dbc->usr = NULL;
1874
1875         list_for_each_entry_safe(bo, bo_temp, &dbc->bo_lists, bo_list) {
1876                 list_for_each_entry_safe(slice, slice_temp, &bo->slices, slice)
1877                         kref_put(&slice->ref_count, free_slice);
1878                 bo->sliced = false;
1879                 INIT_LIST_HEAD(&bo->slices);
1880                 bo->total_slice_nents = 0;
1881                 bo->dir = 0;
1882                 bo->dbc = NULL;
1883                 bo->nr_slice = 0;
1884                 bo->nr_slice_xfer_done = 0;
1885                 bo->queued = false;
1886                 bo->req_id = 0;
1887                 init_completion(&bo->xfer_done);
1888                 complete_all(&bo->xfer_done);
1889                 list_del(&bo->bo_list);
1890                 bo->perf_stats.req_received_ts = 0;
1891                 bo->perf_stats.req_submit_ts = 0;
1892                 bo->perf_stats.req_processed_ts = 0;
1893                 bo->perf_stats.queue_level_before = 0;
1894         }
1895
1896         dbc->in_use = false;
1897         wake_up(&dbc->dbc_release);
1898 }