freedreno/msm/msm_ringbuffer.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2013 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include <config.h>
  31 #endif
  32
  33 #include <assert.h>
  34 #include <inttypes.h>
  35
  36 #include "freedreno_ringbuffer.h"
  37 #include "msm_priv.h"
  38
  39 struct msm_ringbuffer {
  40         struct fd_ringbuffer base;
  41         struct fd_bo *ring_bo;
  42
  43         /* submit ioctl related tables: */
  44         struct {
  45                 /* bo's table: */
  46                 struct drm_msm_gem_submit_bo *bos;
  47                 uint32_t nr_bos, max_bos;
  48
  49                 /* cmd's table: */
  50                 struct drm_msm_gem_submit_cmd *cmds;
  51                 uint32_t nr_cmds, max_cmds;
  52
  53                 /* reloc's table: */
  54                 struct drm_msm_gem_submit_reloc *relocs;
  55                 uint32_t nr_relocs, max_relocs;
  56         } submit;
  57
  58         /* should have matching entries in submit.bos: */
  59         struct fd_bo **bos;
  60         uint32_t nr_bos, max_bos;
  61
  62         /* should have matching entries in submit.cmds: */
  63         struct fd_ringbuffer **rings;
  64         uint32_t nr_rings, max_rings;
  65 };
  66
  67 static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
  68
  69 static void *grow(void *ptr, uint32_t nr, uint32_t *max, uint32_t sz)
  70 {
  71         if ((nr + 1) > *max) {
  72                 if ((*max * 2) < (nr + 1))
  73                         *max = nr + 5;
  74                 else
  75                         *max = *max * 2;
  76                 ptr = realloc(ptr, *max * sz);
  77         }
  78         return ptr;
  79 }
  80
  81 #define APPEND(x, name) ({ \
  82         (x)->name = grow((x)->name, (x)->nr_ ## name, &(x)->max_ ## name, sizeof((x)->name[0])); \
  83         (x)->nr_ ## name ++; \
  84 })
  85
  86 static inline struct msm_ringbuffer * to_msm_ringbuffer(struct fd_ringbuffer *x)
  87 {
  88         return (struct msm_ringbuffer *)x;
  89 }
  90
  91 static uint32_t append_bo(struct fd_ringbuffer *ring, struct fd_bo *bo)
  92 {
  93         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
  94         uint32_t idx;
  95
  96         idx = APPEND(&msm_ring->submit, bos);
  97         idx = APPEND(msm_ring, bos);
  98
  99         msm_ring->submit.bos[idx].flags = 0;
 100         msm_ring->submit.bos[idx].handle = bo->handle;
 101         msm_ring->submit.bos[idx].presumed = to_msm_bo(bo)->presumed;
 102
 103         msm_ring->bos[idx] = fd_bo_ref(bo);
 104
 105         return idx;
 106 }
 107
 108 /* add (if needed) bo, return idx: */
 109 static uint32_t bo2idx(struct fd_ringbuffer *ring, struct fd_bo *bo, uint32_t flags)
 110 {
 111         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 112         struct msm_bo *msm_bo = to_msm_bo(bo);
 113         uint32_t idx;
 114         pthread_mutex_lock(&idx_lock);
 115         if (!msm_bo->current_ring) {
 116                 idx = append_bo(ring, bo);
 117                 msm_bo->current_ring = ring;
 118                 msm_bo->idx = idx;
 119         } else if (msm_bo->current_ring == ring) {
 120                 idx = msm_bo->idx;
 121         } else {
 122                 /* slow-path: */
 123                 for (idx = 0; idx < msm_ring->nr_bos; idx++)
 124                         if (msm_ring->bos[idx] == bo)
 125                                 break;
 126                 if (idx == msm_ring->nr_bos) {
 127                         /* not found */
 128                         idx = append_bo(ring, bo);
 129                 }
 130         }
 131         pthread_mutex_unlock(&idx_lock);
 132         if (flags & FD_RELOC_READ)
 133                 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_READ;
 134         if (flags & FD_RELOC_WRITE)
 135                 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
 136         return idx;
 137 }
 138
 139 static int check_cmd_bo(struct fd_ringbuffer *ring,
 140                 struct drm_msm_gem_submit_cmd *cmd, struct fd_bo *bo)
 141 {
 142         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 143         return msm_ring->submit.bos[cmd->submit_idx].handle == bo->handle;
 144 }
 145
 146 static uint32_t offset_bytes(void *end, void *start)
 147 {
 148         return ((char *)end) - ((char *)start);
 149 }
 150
 151 static struct drm_msm_gem_submit_cmd * get_cmd(struct fd_ringbuffer *ring,
 152                 struct fd_ringbuffer *target_ring, struct fd_bo *target_bo,
 153                 uint32_t submit_offset, uint32_t size, uint32_t type)
 154 {
 155         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 156         struct drm_msm_gem_submit_cmd *cmd = NULL;
 157         uint32_t i;
 158
 159         /* figure out if we already have a cmd buf: */
 160         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 161                 cmd = &msm_ring->submit.cmds[i];
 162                 if ((cmd->submit_offset == submit_offset) &&
 163                                 (cmd->size == size) &&
 164                                 (cmd->type == type) &&
 165                                 check_cmd_bo(ring, cmd, target_bo))
 166                         break;
 167                 cmd = NULL;
 168         }
 169
 170         /* create cmd buf if not: */
 171         if (!cmd) {
 172                 uint32_t idx = APPEND(&msm_ring->submit, cmds);
 173                 APPEND(msm_ring, rings);
 174                 msm_ring->rings[idx] = target_ring;
 175                 cmd = &msm_ring->submit.cmds[idx];
 176                 cmd->type = type;
 177                 cmd->submit_idx = bo2idx(ring, target_bo, FD_RELOC_READ);
 178                 cmd->submit_offset = submit_offset;
 179                 cmd->size = size;
 180                 cmd->pad = 0;
 181         }
 182
 183         return cmd;
 184 }
 185
 186 static void * msm_ringbuffer_hostptr(struct fd_ringbuffer *ring)
 187 {
 188         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 189         return fd_bo_map(msm_ring->ring_bo);
 190 }
 191
 192 static uint32_t find_next_reloc_idx(struct msm_ringbuffer *msm_ring,
 193                 uint32_t start, uint32_t offset)
 194 {
 195         uint32_t i;
 196
 197         /* a binary search would be more clever.. */
 198         for (i = start; i < msm_ring->submit.nr_relocs; i++) {
 199                 struct drm_msm_gem_submit_reloc *reloc = &msm_ring->submit.relocs[i];
 200                 if (reloc->submit_offset >= offset)
 201                         return i;
 202         }
 203
 204         return i;
 205 }
 206
 207 static void flush_reset(struct fd_ringbuffer *ring)
 208 {
 209         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 210         unsigned i;
 211
 212         /* for each of the cmd buffers, clear their reloc's: */
 213         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 214                 struct msm_ringbuffer *target_ring = to_msm_ringbuffer(msm_ring->rings[i]);
 215                 target_ring->submit.nr_relocs = 0;
 216         }
 217
 218         msm_ring->submit.nr_relocs = 0;
 219         msm_ring->submit.nr_cmds = 0;
 220         msm_ring->submit.nr_bos = 0;
 221         msm_ring->nr_rings = 0;
 222         msm_ring->nr_bos = 0;
 223 }
 224
 225 static int msm_ringbuffer_flush(struct fd_ringbuffer *ring, uint32_t *last_start)
 226 {
 227         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 228         struct fd_bo *ring_bo = msm_ring->ring_bo;
 229         struct drm_msm_gem_submit req = {
 230                         .pipe = to_msm_pipe(ring->pipe)->pipe,
 231         };
 232         uint32_t i, j, submit_offset, size;
 233         int ret;
 234
 235         submit_offset = offset_bytes(last_start, ring->start);
 236         size = offset_bytes(ring->cur, last_start);
 237
 238         get_cmd(ring, ring, ring_bo, submit_offset, size, MSM_SUBMIT_CMD_BUF);
 239
 240         /* needs to be after get_cmd() as that could create bos/cmds table: */
 241         req.bos = VOID2U64(msm_ring->submit.bos),
 242         req.nr_bos = msm_ring->submit.nr_bos;
 243         req.cmds = VOID2U64(msm_ring->submit.cmds),
 244         req.nr_cmds = msm_ring->submit.nr_cmds;
 245
 246         /* for each of the cmd's fix up their reloc's: */
 247         for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 248                 struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i];
 249                 struct msm_ringbuffer *target_ring = to_msm_ringbuffer(msm_ring->rings[i]);
 250                 uint32_t a = find_next_reloc_idx(target_ring, 0, cmd->submit_offset);
 251                 uint32_t b = find_next_reloc_idx(target_ring, a, cmd->submit_offset + cmd->size);
 252                 cmd->relocs = VOID2U64(&target_ring->submit.relocs[a]);
 253                 cmd->nr_relocs = (b > a) ? b - a : 0;
 254         }
 255
 256         DEBUG_MSG("nr_cmds=%u, nr_bos=%u\n", req.nr_cmds, req.nr_bos);
 257
 258         ret = drmCommandWriteRead(ring->pipe->dev->fd, DRM_MSM_GEM_SUBMIT,
 259                         &req, sizeof(req));
 260         if (ret) {
 261                 ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
 262                 ERROR_MSG("  pipe:  %u", req.pipe);
 263                 for (i = 0; i < msm_ring->submit.nr_bos; i++) {
 264                         struct drm_msm_gem_submit_bo *bo = &msm_ring->submit.bos[i];
 265                         ERROR_MSG("  bos[%d]: handle=%u, flags=%x", i, bo->handle, bo->flags);
 266                 }
 267                 for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 268                         struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i];
 269                         struct drm_msm_gem_submit_reloc *relocs = U642VOID(cmd->relocs);
 270                         ERROR_MSG("  cmd[%d]: type=%u, submit_idx=%u, submit_offset=%u, size=%u\n",
 271                                         i, cmd->type, cmd->submit_idx, cmd->submit_offset, cmd->size);
 272                         for (j = 0; j < cmd->nr_relocs; j++) {
 273                                 struct drm_msm_gem_submit_reloc *r = &relocs[j];
 274                                 ERROR_MSG("    reloc[%d]: submit_offset=%u, or=%08x, shift=%d, reloc_idx=%u"
 275                                                 ", reloc_offset=%"PRIu64, j, r->submit_offset, r->or, r->shift,
 276                                                 r->reloc_idx, r->reloc_offset);
 277                         }
 278                 }
 279         } else {
 280                 /* update timestamp on all rings associated with submit: */
 281                 for (i = 0; i < msm_ring->submit.nr_cmds; i++) {
 282                         struct fd_ringbuffer *target_ring = msm_ring->rings[i];
 283                         if (!ret)
 284                                 target_ring->last_timestamp = req.fence;
 285                 }
 286         }
 287
 288         for (i = 0; i < msm_ring->nr_bos; i++) {
 289                 struct msm_bo *msm_bo = to_msm_bo(msm_ring->bos[i]);
 290                 msm_bo->current_ring = NULL;
 291                 fd_bo_del(&msm_bo->base);
 292         }
 293
 294         flush_reset(ring);
 295
 296         return ret;
 297 }
 298
 299 static void msm_ringbuffer_reset(struct fd_ringbuffer *ring)
 300 {
 301         flush_reset(ring);
 302 }
 303
 304 static void msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
 305                 const struct fd_reloc *r)
 306 {
 307         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 308         struct fd_ringbuffer *parent = ring->parent ? ring->parent : ring;
 309         struct msm_bo *msm_bo = to_msm_bo(r->bo);
 310         struct drm_msm_gem_submit_reloc *reloc;
 311         uint32_t idx = APPEND(&msm_ring->submit, relocs);
 312         uint32_t addr;
 313
 314         reloc = &msm_ring->submit.relocs[idx];
 315
 316         reloc->reloc_idx = bo2idx(parent, r->bo, r->flags);
 317         reloc->reloc_offset = r->offset;
 318         reloc->or = r->or;
 319         reloc->shift = r->shift;
 320         reloc->submit_offset = offset_bytes(ring->cur, ring->start);
 321
 322         addr = msm_bo->presumed;
 323         if (r->shift < 0)
 324                 addr >>= -r->shift;
 325         else
 326                 addr <<= r->shift;
 327         (*ring->cur++) = addr | r->or;
 328 }
 329
 330 static void msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
 331                 struct fd_ringmarker *target, struct fd_ringmarker *end)
 332 {
 333         struct fd_bo *target_bo = to_msm_ringbuffer(target->ring)->ring_bo;
 334         struct drm_msm_gem_submit_cmd *cmd;
 335         uint32_t submit_offset, size;
 336
 337         submit_offset = offset_bytes(target->cur, target->ring->start);
 338         size = offset_bytes(end->cur, target->cur);
 339
 340         cmd = get_cmd(ring, target->ring, target_bo, submit_offset, size,
 341                         MSM_SUBMIT_CMD_IB_TARGET_BUF);
 342         assert(cmd);
 343
 344         msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
 345                 .bo = target_bo,
 346                 .flags = FD_RELOC_READ,
 347                 .offset = submit_offset,
 348         });
 349 }
 350
 351 static void msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
 352 {
 353         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 354         if (msm_ring->ring_bo)
 355                 fd_bo_del(msm_ring->ring_bo);
 356         free(msm_ring);
 357 }
 358
 359 static struct fd_ringbuffer_funcs funcs = {
 360                 .hostptr = msm_ringbuffer_hostptr,
 361                 .flush = msm_ringbuffer_flush,
 362                 .reset = msm_ringbuffer_reset,
 363                 .emit_reloc = msm_ringbuffer_emit_reloc,
 364                 .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
 365                 .destroy = msm_ringbuffer_destroy,
 366 };
 367
 368 drm_private struct fd_ringbuffer * msm_ringbuffer_new(struct fd_pipe *pipe,
 369                 uint32_t size)
 370 {
 371         struct msm_ringbuffer *msm_ring;
 372         struct fd_ringbuffer *ring = NULL;
 373
 374         msm_ring = calloc(1, sizeof(*msm_ring));
 375         if (!msm_ring) {
 376                 ERROR_MSG("allocation failed");
 377                 goto fail;
 378         }
 379
 380         ring = &msm_ring->base;
 381         ring->funcs = &funcs;
 382
 383         msm_ring->ring_bo = fd_bo_new(pipe->dev, size, 0);
 384         if (!msm_ring->ring_bo) {
 385                 ERROR_MSG("ringbuffer allocation failed");
 386                 goto fail;
 387         }
 388
 389         return ring;
 390 fail:
 391         if (ring)
 392                 fd_ringbuffer_del(ring);
 393         return NULL;
 394 }