freedreno/msm/msm_ringbuffer.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2013 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include <config.h>
  31 #endif
  32
  33 #include <assert.h>
  34 #include <inttypes.h>
  35
  36 #include "freedreno_ringbuffer.h"
  37 #include "msm_priv.h"
  38
  39 /* represents a single cmd buffer in the submit ioctl.  Each cmd buffer has
  40  * a backing bo, and a reloc table.
  41  */
  42 struct msm_cmd {
  43         struct list_head list;
  44
  45         struct fd_ringbuffer *ring;
  46         struct fd_bo *ring_bo;
  47
  48         /* reloc's table: */
  49         struct drm_msm_gem_submit_reloc *relocs;
  50         uint32_t nr_relocs, max_relocs;
  51
  52         uint32_t size;
  53 };
  54
  55 struct msm_ringbuffer {
  56         struct fd_ringbuffer base;
  57
  58         /* submit ioctl related tables:
  59          * Note that bos and cmds are tracked by the parent ringbuffer, since
  60          * that is global to the submit ioctl call.  The reloc's table is tracked
  61          * per cmd-buffer.
  62          */
  63         struct {
  64                 /* bo's table: */
  65                 struct drm_msm_gem_submit_bo *bos;
  66                 uint32_t nr_bos, max_bos;
  67
  68                 /* cmd's table: */
  69                 struct drm_msm_gem_submit_cmd *cmds;
  70                 uint32_t nr_cmds, max_cmds;
  71         } submit;
  72
  73         /* should have matching entries in submit.bos: */
  74         /* Note, only in parent ringbuffer */
  75         struct fd_bo **bos;
  76         uint32_t nr_bos, max_bos;
  77
  78         /* should have matching entries in submit.cmds: */
  79         struct msm_cmd **cmds;
  80         uint32_t nr_cmds, max_cmds;
  81
  82         /* List of physical cmdstream buffers (msm_cmd) assocated with this
  83          * logical fd_ringbuffer.
  84          *
  85          * Note that this is different from msm_ringbuffer::cmds (which
  86          * shadows msm_ringbuffer::submit::cmds for tracking submit ioctl
  87          * related stuff, and *only* is tracked in the parent ringbuffer.
  88          * And only has "completed" cmd buffers (ie. we already know the
  89          * size) added via get_cmd().
  90          */
  91         struct list_head cmd_list;
  92
  93         int is_growable;
  94         unsigned cmd_count;
  95 };
  96
  97 static inline struct msm_ringbuffer * to_msm_ringbuffer(struct fd_ringbuffer *x)
  98 {
  99         return (struct msm_ringbuffer *)x;
 100 }
 101
 102 #define INIT_SIZE 0x1000
 103
 104 static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
 105 drm_private extern pthread_mutex_t table_lock;
 106
 107 static void ring_bo_del(struct fd_device *dev, struct fd_bo *bo)
 108 {
 109         int ret;
 110
 111         pthread_mutex_lock(&table_lock);
 112         ret = fd_bo_cache_free(&to_msm_device(dev)->ring_cache, bo);
 113         pthread_mutex_unlock(&table_lock);
 114
 115         if (ret == 0)
 116                 return;
 117
 118         fd_bo_del(bo);
 119 }
 120
 121 static struct fd_bo * ring_bo_new(struct fd_device *dev, uint32_t size)
 122 {
 123         struct fd_bo *bo;
 124
 125         bo = fd_bo_cache_alloc(&to_msm_device(dev)->ring_cache, &size, 0);
 126         if (bo)
 127                 return bo;
 128
 129         bo = fd_bo_new(dev, size, 0);
 130         if (!bo)
 131                 return NULL;
 132
 133         /* keep ringbuffer bo's out of the normal bo cache: */
 134         bo->bo_reuse = FALSE;
 135
 136         return bo;
 137 }
 138
 139 static void ring_cmd_del(struct msm_cmd *cmd)
 140 {
 141         if (cmd->ring_bo)
 142                 ring_bo_del(cmd->ring->pipe->dev, cmd->ring_bo);
 143         list_del(&cmd->list);
 144         to_msm_ringbuffer(cmd->ring)->cmd_count--;
 145         free(cmd->relocs);
 146         free(cmd);
 147 }
 148
 149 static struct msm_cmd * ring_cmd_new(struct fd_ringbuffer *ring, uint32_t size)
 150 {
 151         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 152         struct msm_cmd *cmd = calloc(1, sizeof(*cmd));
 153
 154         if (!cmd)
 155                 return NULL;
 156
 157         cmd->ring = ring;
 158         cmd->ring_bo = ring_bo_new(ring->pipe->dev, size);
 159         if (!cmd->ring_bo)
 160                 goto fail;
 161
 162         list_addtail(&cmd->list, &msm_ring->cmd_list);
 163         msm_ring->cmd_count++;
 164
 165         return cmd;
 166
 167 fail:
 168         ring_cmd_del(cmd);
 169         return NULL;
 170 }
 171
 172 static void *grow(void *ptr, uint32_t nr, uint32_t *max, uint32_t sz)
 173 {
 174         if ((nr + 1) > *max) {
 175                 if ((*max * 2) < (nr + 1))
 176                         *max = nr + 5;
 177                 else
 178                         *max = *max * 2;
 179                 ptr = realloc(ptr, *max * sz);
 180         }
 181         return ptr;
 182 }
 183
 184 #define APPEND(x, name) ({ \
 185         (x)->name = grow((x)->name, (x)->nr_ ## name, &(x)->max_ ## name, sizeof((x)->name[0])); \
 186         (x)->nr_ ## name ++; \
 187 })
 188
 189 static struct msm_cmd *current_cmd(struct fd_ringbuffer *ring)
 190 {
 191         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 192         assert(!LIST_IS_EMPTY(&msm_ring->cmd_list));
 193         return LIST_LAST_ENTRY(&msm_ring->cmd_list, struct msm_cmd, list);
 194 }
 195
 196 static uint32_t append_bo(struct fd_ringbuffer *ring, struct fd_bo *bo)
 197 {
 198         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 199         uint32_t idx;
 200
 201         idx = APPEND(&msm_ring->submit, bos);
 202         idx = APPEND(msm_ring, bos);
 203
 204         msm_ring->submit.bos[idx].flags = 0;
 205         msm_ring->submit.bos[idx].handle = bo->handle;
 206         msm_ring->submit.bos[idx].presumed = to_msm_bo(bo)->presumed;
 207
 208         msm_ring->bos[idx] = fd_bo_ref(bo);
 209
 210         return idx;
 211 }
 212
 213 /* add (if needed) bo, return idx: */
 214 static uint32_t bo2idx(struct fd_ringbuffer *ring, struct fd_bo *bo, uint32_t flags)
 215 {
 216         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 217         struct msm_bo *msm_bo = to_msm_bo(bo);
 218         uint32_t idx;
 219         pthread_mutex_lock(&idx_lock);
 220         if (!msm_bo->current_ring) {
 221                 idx = append_bo(ring, bo);
 222                 msm_bo->current_ring = ring;
 223                 msm_bo->idx = idx;
 224         } else if (msm_bo->current_ring == ring) {
 225                 idx = msm_bo->idx;
 226         } else {
 227                 /* slow-path: */
 228                 for (idx = 0; idx < msm_ring->nr_bos; idx++)
 229                         if (msm_ring->bos[idx] == bo)
 230                                 break;
 231                 if (idx == msm_ring->nr_bos) {
 232                         /* not found */
 233                         idx = append_bo(ring, bo);
 234                 }
 235         }
 236         pthread_mutex_unlock(&idx_lock);
 237         if (flags & FD_RELOC_READ)
 238                 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_READ;
 239         if (flags & FD_RELOC_WRITE)
 240                 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
 241         return idx;
 242 }
 243
 244 static int check_cmd_bo(struct fd_ringbuffer *ring,
 245                 struct drm_msm_gem_submit_cmd *cmd, struct fd_bo *bo)
 246 {
 247         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 248         return msm_ring->submit.bos[cmd->submit_idx].handle == bo->handle;
 249 }
 250
 251 /* Ensure that submit has corresponding entry in cmds table for the
 252  * target cmdstream buffer:
 253  */
 254 static void get_cmd(struct fd_ringbuffer *ring, struct msm_cmd *target_cmd,
 255                 uint32_t submit_offset, uint32_t size, uint32_t type)
 256 {
 257         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 258         struct drm_msm_gem_submit_cmd *cmd;
 259         uint32_t i;
 260
 261         /* figure out if we already have a cmd buf: */
 262         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 263                 cmd = &msm_ring->submit.cmds[i];
 264                 if ((cmd->submit_offset == submit_offset) &&
 265                                 (cmd->size == size) &&
 266                                 (cmd->type == type) &&
 267                                 check_cmd_bo(ring, cmd, target_cmd->ring_bo))
 268                         return;
 269         }
 270
 271         /* create cmd buf if not: */
 272         i = APPEND(&msm_ring->submit, cmds);
 273         APPEND(msm_ring, cmds);
 274         msm_ring->cmds[i] = target_cmd;
 275         cmd = &msm_ring->submit.cmds[i];
 276         cmd->type = type;
 277         cmd->submit_idx = bo2idx(ring, target_cmd->ring_bo, FD_RELOC_READ);
 278         cmd->submit_offset = submit_offset;
 279         cmd->size = size;
 280         cmd->pad = 0;
 281
 282         target_cmd->size = size;
 283 }
 284
 285 static void * msm_ringbuffer_hostptr(struct fd_ringbuffer *ring)
 286 {
 287         return fd_bo_map(current_cmd(ring)->ring_bo);
 288 }
 289
 290 static uint32_t find_next_reloc_idx(struct msm_cmd *msm_cmd,
 291                 uint32_t start, uint32_t offset)
 292 {
 293         uint32_t i;
 294
 295         /* a binary search would be more clever.. */
 296         for (i = start; i < msm_cmd->nr_relocs; i++) {
 297                 struct drm_msm_gem_submit_reloc *reloc = &msm_cmd->relocs[i];
 298                 if (reloc->submit_offset >= offset)
 299                         return i;
 300         }
 301
 302         return i;
 303 }
 304
 305 static void delete_cmds(struct msm_ringbuffer *msm_ring)
 306 {
 307         struct msm_cmd *cmd, *tmp;
 308
 309         LIST_FOR_EACH_ENTRY_SAFE(cmd, tmp, &msm_ring->cmd_list, list) {
 310                 ring_cmd_del(cmd);
 311         }
 312 }
 313
 314 static void flush_reset(struct fd_ringbuffer *ring)
 315 {
 316         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 317         unsigned i;
 318
 319         for (i = 0; i < msm_ring->nr_bos; i++) {
 320                 struct msm_bo *msm_bo = to_msm_bo(msm_ring->bos[i]);
 321                 msm_bo->current_ring = NULL;
 322                 fd_bo_del(&msm_bo->base);
 323         }
 324
 325         /* for each of the cmd buffers, clear their reloc's: */
 326         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 327                 struct msm_cmd *target_cmd = msm_ring->cmds[i];
 328                 target_cmd->nr_relocs = 0;
 329         }
 330
 331         msm_ring->submit.nr_cmds = 0;
 332         msm_ring->submit.nr_bos = 0;
 333         msm_ring->nr_cmds = 0;
 334         msm_ring->nr_bos = 0;
 335
 336         if (msm_ring->is_growable) {
 337                 delete_cmds(msm_ring);
 338         } else {
 339                 /* in old mode, just reset the # of relocs: */
 340                 current_cmd(ring)->nr_relocs = 0;
 341         }
 342 }
 343
 344 static void finalize_current_cmd(struct fd_ringbuffer *ring, uint32_t *last_start)
 345 {
 346         uint32_t submit_offset, size, type;
 347         struct fd_ringbuffer *parent;
 348
 349         if (ring->parent) {
 350                 parent = ring->parent;
 351                 type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 352         } else {
 353                 parent = ring;
 354                 type = MSM_SUBMIT_CMD_BUF;
 355         }
 356
 357         submit_offset = offset_bytes(last_start, ring->start);
 358         size = offset_bytes(ring->cur, last_start);
 359
 360         get_cmd(parent, current_cmd(ring), submit_offset, size, type);
 361 }
 362
 363 static void dump_submit(struct msm_ringbuffer *msm_ring)
 364 {
 365         uint32_t i, j;
 366
 367         for (i = 0; i < msm_ring->submit.nr_bos; i++) {
 368                 struct drm_msm_gem_submit_bo *bo = &msm_ring->submit.bos[i];
 369                 ERROR_MSG("  bos[%d]: handle=%u, flags=%x", i, bo->handle, bo->flags);
 370         }
 371         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 372                 struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i];
 373                 struct drm_msm_gem_submit_reloc *relocs = U642VOID(cmd->relocs);
 374                 ERROR_MSG("  cmd[%d]: type=%u, submit_idx=%u, submit_offset=%u, size=%u",
 375                                 i, cmd->type, cmd->submit_idx, cmd->submit_offset, cmd->size);
 376                 for (j = 0; j < cmd->nr_relocs; j++) {
 377                         struct drm_msm_gem_submit_reloc *r = &relocs[j];
 378                         ERROR_MSG("    reloc[%d]: submit_offset=%u, or=%08x, shift=%d, reloc_idx=%u"
 379                                         ", reloc_offset=%"PRIu64, j, r->submit_offset, r->or, r->shift,
 380                                         r->reloc_idx, r->reloc_offset);
 381                 }
 382         }
 383 }
 384
 385 static int msm_ringbuffer_flush(struct fd_ringbuffer *ring, uint32_t *last_start)
 386 {
 387         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 388         struct drm_msm_gem_submit req = {
 389                         .pipe = to_msm_pipe(ring->pipe)->pipe,
 390         };
 391         uint32_t i;
 392         int ret;
 393
 394         finalize_current_cmd(ring, last_start);
 395
 396         /* needs to be after get_cmd() as that could create bos/cmds table: */
 397         req.bos = VOID2U64(msm_ring->submit.bos),
 398         req.nr_bos = msm_ring->submit.nr_bos;
 399         req.cmds = VOID2U64(msm_ring->submit.cmds),
 400         req.nr_cmds = msm_ring->submit.nr_cmds;
 401
 402         /* for each of the cmd's fix up their reloc's: */
 403         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 404                 struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i];
 405                 struct msm_cmd *msm_cmd = msm_ring->cmds[i];
 406                 uint32_t a = find_next_reloc_idx(msm_cmd, 0, cmd->submit_offset);
 407                 uint32_t b = find_next_reloc_idx(msm_cmd, a, cmd->submit_offset + cmd->size);
 408                 cmd->relocs = VOID2U64(&msm_cmd->relocs[a]);
 409                 cmd->nr_relocs = (b > a) ? b - a : 0;
 410         }
 411
 412         DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
 413
 414         ret = drmCommandWriteRead(ring->pipe->dev->fd, DRM_MSM_GEM_SUBMIT,
 415                         &req, sizeof(req));
 416         if (ret) {
 417                 ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
 418                 dump_submit(msm_ring);
 419         } else if (!ret) {
 420                 /* update timestamp on all rings associated with submit: */
 421                 for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 422                         struct msm_cmd *msm_cmd = msm_ring->cmds[i];
 423                         msm_cmd->ring->last_timestamp = req.fence;
 424                 }
 425         }
 426
 427         flush_reset(ring);
 428
 429         return ret;
 430 }
 431
 432 static void msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size)
 433 {
 434         assert(to_msm_ringbuffer(ring)->is_growable);
 435         finalize_current_cmd(ring, ring->last_start);
 436         ring_cmd_new(ring, size);
 437 }
 438
 439 static void msm_ringbuffer_reset(struct fd_ringbuffer *ring)
 440 {
 441         flush_reset(ring);
 442 }
 443
 444 static void msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
 445                 const struct fd_reloc *r)
 446 {
 447         struct fd_ringbuffer *parent = ring->parent ? ring->parent : ring;
 448         struct msm_bo *msm_bo = to_msm_bo(r->bo);
 449         struct drm_msm_gem_submit_reloc *reloc;
 450         struct msm_cmd *cmd = current_cmd(ring);
 451         uint32_t idx = APPEND(cmd, relocs);
 452         uint32_t addr;
 453
 454         reloc = &cmd->relocs[idx];
 455
 456         reloc->reloc_idx = bo2idx(parent, r->bo, r->flags);
 457         reloc->reloc_offset = r->offset;
 458         reloc->or = r->or;
 459         reloc->shift = r->shift;
 460         reloc->submit_offset = offset_bytes(ring->cur, ring->start);
 461
 462         addr = msm_bo->presumed;
 463         if (r->shift < 0)
 464                 addr >>= -r->shift;
 465         else
 466                 addr <<= r->shift;
 467         (*ring->cur++) = addr | r->or;
 468 }
 469
 470 static uint32_t msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
 471                 struct fd_ringbuffer *target, uint32_t cmd_idx,
 472                 uint32_t submit_offset, uint32_t size)
 473 {
 474         struct msm_cmd *cmd = NULL;
 475         uint32_t idx = 0;
 476
 477         LIST_FOR_EACH_ENTRY(cmd, &to_msm_ringbuffer(target)->cmd_list, list) {
 478                 if (idx == cmd_idx)
 479                         break;
 480                 idx++;
 481         }
 482
 483         assert(cmd && (idx == cmd_idx));
 484
 485         if (idx < (to_msm_ringbuffer(target)->cmd_count - 1)) {
 486                 /* All but the last cmd buffer is fully "baked" (ie. already has
 487                  * done get_cmd() to add it to the cmds table).  But in this case,
 488                  * the size we get is invalid (since it is calculated from the
 489                  * last cmd buffer):
 490                  */
 491                 size = cmd->size;
 492         } else {
 493                 get_cmd(ring, cmd, submit_offset, size, MSM_SUBMIT_CMD_IB_TARGET_BUF);
 494         }
 495
 496         msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
 497                 .bo = cmd->ring_bo,
 498                 .flags = FD_RELOC_READ,
 499                 .offset = submit_offset,
 500         });
 501
 502         return size;
 503 }
 504
 505 static uint32_t msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
 506 {
 507         return to_msm_ringbuffer(ring)->cmd_count;
 508 }
 509
 510 static void msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
 511 {
 512         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 513
 514         flush_reset(ring);
 515         delete_cmds(msm_ring);
 516
 517         free(msm_ring->submit.cmds);
 518         free(msm_ring->submit.bos);
 519         free(msm_ring->bos);
 520         free(msm_ring->cmds);
 521         free(msm_ring);
 522 }
 523
 524 static const struct fd_ringbuffer_funcs funcs = {
 525                 .hostptr = msm_ringbuffer_hostptr,
 526                 .flush = msm_ringbuffer_flush,
 527                 .grow = msm_ringbuffer_grow,
 528                 .reset = msm_ringbuffer_reset,
 529                 .emit_reloc = msm_ringbuffer_emit_reloc,
 530                 .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
 531                 .cmd_count = msm_ringbuffer_cmd_count,
 532                 .destroy = msm_ringbuffer_destroy,
 533 };
 534
 535 drm_private struct fd_ringbuffer * msm_ringbuffer_new(struct fd_pipe *pipe,
 536                 uint32_t size)
 537 {
 538         struct msm_ringbuffer *msm_ring;
 539         struct fd_ringbuffer *ring = NULL;
 540
 541         msm_ring = calloc(1, sizeof(*msm_ring));
 542         if (!msm_ring) {
 543                 ERROR_MSG("allocation failed");
 544                 goto fail;
 545         }
 546
 547         if (size == 0) {
 548                 assert(pipe->dev->version >= FD_VERSION_UNLIMITED_CMDS);
 549                 size = INIT_SIZE;
 550                 msm_ring->is_growable = TRUE;
 551         }
 552
 553         list_inithead(&msm_ring->cmd_list);
 554
 555         ring = &msm_ring->base;
 556         ring->funcs = &funcs;
 557         ring->size = size;
 558         ring->pipe = pipe;   /* needed in ring_cmd_new() */
 559
 560         ring_cmd_new(ring, size);
 561
 562         return ring;
 563 fail:
 564         if (ring)
 565                 fd_ringbuffer_del(ring);
 566         return NULL;
 567 }