intel/intel_bufmgr_fake.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /* Originally a fake version of the buffer manager so that we can
  29  * prototype the changes in a driver fairly quickly, has been fleshed
  30  * out to a fully functional interim solution.
  31  *
  32  * Basically wraps the old style memory management in the new
  33  * programming interface, but is more expressive and avoids many of
  34  * the bugs in the old texture manager.
  35  */
  36
  37 #ifdef HAVE_CONFIG_H
  38 #include "config.h"
  39 #endif
  40
  41 #include <stdlib.h>
  42 #include <string.h>
  43 #include <assert.h>
  44 #include <errno.h>
  45 #include <xf86drm.h>
  46 #include <pthread.h>
  47 #include "intel_bufmgr.h"
  48 #include "intel_bufmgr_priv.h"
  49 #include "drm.h"
  50 #include "i915_drm.h"
  51 #include "mm.h"
  52 #include "libdrm_lists.h"
  53
  54 #define DBG(...) do {                                   \
  55         if (bufmgr_fake->bufmgr.debug)                  \
  56                 drmMsg(__VA_ARGS__);                    \
  57 } while (0)
  58
  59 /* Internal flags:
  60  */
  61 #define BM_NO_BACKING_STORE                     0x00000001
  62 #define BM_NO_FENCE_SUBDATA                     0x00000002
  63 #define BM_PINNED                               0x00000004
  64
  65 /* Wrapper around mm.c's mem_block, which understands that you must
  66  * wait for fences to expire before memory can be freed.  This is
  67  * specific to our use of memcpy for uploads - an upload that was
  68  * processed through the command queue wouldn't need to care about
  69  * fences.
  70  */
  71 #define MAX_RELOCS 4096
  72
  73 struct fake_buffer_reloc {
  74         /** Buffer object that the relocation points at. */
  75         drm_intel_bo *target_buf;
  76         /** Offset of the relocation entry within reloc_buf. */
  77         uint32_t offset;
  78         /**
  79          * Cached value of the offset when we last performed this relocation.
  80          */
  81         uint32_t last_target_offset;
  82         /** Value added to target_buf's offset to get the relocation entry. */
  83         uint32_t delta;
  84         /** Cache domains the target buffer is read into. */
  85         uint32_t read_domains;
  86         /** Cache domain the target buffer will have dirty cachelines in. */
  87         uint32_t write_domain;
  88 };
  89
  90 struct block {
  91         struct block *next, *prev;
  92         struct mem_block *mem;  /* BM_MEM_AGP */
  93
  94         /**
  95          * Marks that the block is currently in the aperture and has yet to be
  96          * fenced.
  97          */
  98         unsigned on_hardware:1;
  99         /**
 100          * Marks that the block is currently fenced (being used by rendering)
 101          * and can't be freed until @fence is passed.
 102          */
 103         unsigned fenced:1;
 104
 105         /** Fence cookie for the block. */
 106         unsigned fence;         /* Split to read_fence, write_fence */
 107
 108         drm_intel_bo *bo;
 109         void *virtual;
 110 };
 111
 112 typedef struct _bufmgr_fake {
 113         drm_intel_bufmgr bufmgr;
 114
 115         pthread_mutex_t lock;
 116
 117         unsigned long low_offset;
 118         unsigned long size;
 119         void *virtual;
 120
 121         struct mem_block *heap;
 122
 123         unsigned buf_nr;        /* for generating ids */
 124
 125         /**
 126          * List of blocks which are currently in the GART but haven't been
 127          * fenced yet.
 128          */
 129         struct block on_hardware;
 130         /**
 131          * List of blocks which are in the GART and have an active fence on
 132          * them.
 133          */
 134         struct block fenced;
 135         /**
 136          * List of blocks which have an expired fence and are ready to be
 137          * evicted.
 138          */
 139         struct block lru;
 140
 141         unsigned int last_fence;
 142
 143         unsigned fail:1;
 144         unsigned need_fence:1;
 145         int thrashing;
 146
 147         /**
 148          * Driver callback to emit a fence, returning the cookie.
 149          *
 150          * This allows the driver to hook in a replacement for the DRM usage in
 151          * bufmgr_fake.
 152          *
 153          * Currently, this also requires that a write flush be emitted before
 154          * emitting the fence, but this should change.
 155          */
 156         unsigned int (*fence_emit) (void *private);
 157         /** Driver callback to wait for a fence cookie to have passed. */
 158         void (*fence_wait) (unsigned int fence, void *private);
 159         void *fence_priv;
 160
 161         /**
 162          * Driver callback to execute a buffer.
 163          *
 164          * This allows the driver to hook in a replacement for the DRM usage in
 165          * bufmgr_fake.
 166          */
 167         int (*exec) (drm_intel_bo *bo, unsigned int used, void *priv);
 168         void *exec_priv;
 169
 170         /** Driver-supplied argument to driver callbacks */
 171         void *driver_priv;
 172         /**
 173          * Pointer to kernel-updated sarea data for the last completed user irq
 174          */
 175         volatile int *last_dispatch;
 176
 177         int fd;
 178
 179         int debug;
 180
 181         int performed_rendering;
 182 } drm_intel_bufmgr_fake;
 183
 184 typedef struct _drm_intel_bo_fake {
 185         drm_intel_bo bo;
 186
 187         unsigned id;            /* debug only */
 188         const char *name;
 189
 190         unsigned dirty:1;
 191         /**
 192          * has the card written to this buffer - we make need to copy it back
 193          */
 194         unsigned card_dirty:1;
 195         unsigned int refcount;
 196         /* Flags may consist of any of the DRM_BO flags, plus
 197          * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the
 198          * first two driver private flags.
 199          */
 200         uint64_t flags;
 201         /** Cache domains the target buffer is read into. */
 202         uint32_t read_domains;
 203         /** Cache domain the target buffer will have dirty cachelines in. */
 204         uint32_t write_domain;
 205
 206         unsigned int alignment;
 207         int is_static, validated;
 208         unsigned int map_count;
 209
 210         /** relocation list */
 211         struct fake_buffer_reloc *relocs;
 212         int nr_relocs;
 213         /**
 214          * Total size of the target_bos of this buffer.
 215          *
 216          * Used for estimation in check_aperture.
 217          */
 218         unsigned int child_size;
 219
 220         struct block *block;
 221         void *backing_store;
 222         void (*invalidate_cb) (drm_intel_bo *bo, void *ptr);
 223         void *invalidate_ptr;
 224 } drm_intel_bo_fake;
 225
 226 static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake,
 227                         unsigned int fence_cookie);
 228
 229 #define MAXFENCE 0x7fffffff
 230
 231 static int
 232 FENCE_LTE(unsigned a, unsigned b)
 233 {
 234         if (a == b)
 235                 return 1;
 236
 237         if (a < b && b - a < (1 << 24))
 238                 return 1;
 239
 240         if (a > b && MAXFENCE - a + b < (1 << 24))
 241                 return 1;
 242
 243         return 0;
 244 }
 245
 246 void
 247 drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr,
 248                                          unsigned int (*emit) (void *priv),
 249                                          void (*wait) (unsigned int fence,
 250                                                        void *priv),
 251                                          void *priv)
 252 {
 253         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 254
 255         bufmgr_fake->fence_emit = emit;
 256         bufmgr_fake->fence_wait = wait;
 257         bufmgr_fake->fence_priv = priv;
 258 }
 259
 260 static unsigned int
 261 _fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake)
 262 {
 263         struct drm_i915_irq_emit ie;
 264         int ret, seq = 1;
 265
 266         if (bufmgr_fake->fence_emit != NULL) {
 267                 seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv);
 268                 return seq;
 269         }
 270
 271         ie.irq_seq = &seq;
 272         ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT,
 273                                   &ie, sizeof(ie));
 274         if (ret) {
 275                 drmMsg("%s: drm_i915_irq_emit: %d\n", __FUNCTION__, ret);
 276                 abort();
 277         }
 278
 279         DBG("emit 0x%08x\n", seq);
 280         return seq;
 281 }
 282
 283 static void
 284 _fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq)
 285 {
 286         struct drm_i915_irq_wait iw;
 287         int hw_seq, busy_count = 0;
 288         int ret;
 289         int kernel_lied;
 290
 291         if (bufmgr_fake->fence_wait != NULL) {
 292                 bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv);
 293                 clear_fenced(bufmgr_fake, seq);
 294                 return;
 295         }
 296
 297         DBG("wait 0x%08x\n", iw.irq_seq);
 298
 299         iw.irq_seq = seq;
 300
 301         /* The kernel IRQ_WAIT implementation is all sorts of broken.
 302          * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit
 303          *    unsigned range.
 304          * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit
 305          *    signed range.
 306          * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit
 307          *    signed range.
 308          * 4) It returns -EBUSY in 3 seconds even if the hardware is still
 309          *    successfully chewing through buffers.
 310          *
 311          * Assume that in userland we treat sequence numbers as ints, which
 312          * makes some of the comparisons convenient, since the sequence
 313          * numbers are all postive signed integers.
 314          *
 315          * From this we get several cases we need to handle.  Here's a timeline.
 316          * 0x2   0x7                                    0x7ffffff8   0x7ffffffd
 317          *   |    |                                             |    |
 318          * ------------------------------------------------------------
 319          *
 320          * A) Normal wait for hw to catch up
 321          * hw_seq seq
 322          *   |    |
 323          * ------------------------------------------------------------
 324          * seq - hw_seq = 5.  If we call IRQ_WAIT, it will wait for hw to
 325          * catch up.
 326          *
 327          * B) Normal wait for a sequence number that's already passed.
 328          * seq    hw_seq
 329          *   |    |
 330          * ------------------------------------------------------------
 331          * seq - hw_seq = -5.  If we call IRQ_WAIT, it returns 0 quickly.
 332          *
 333          * C) Hardware has already wrapped around ahead of us
 334          * hw_seq                                                    seq
 335          *   |                                                       |
 336          * ------------------------------------------------------------
 337          * seq - hw_seq = 0x80000000 - 5.  If we called IRQ_WAIT, it would wait
 338          * for hw_seq >= seq, which may never occur.  Thus, we want to catch
 339          * this in userland and return 0.
 340          *
 341          * D) We've wrapped around ahead of the hardware.
 342          * seq                                                      hw_seq
 343          *   |                                                       |
 344          * ------------------------------------------------------------
 345          * seq - hw_seq = -(0x80000000 - 5).  If we called IRQ_WAIT, it would
 346          * return 0 quickly because hw_seq >= seq, even though the hardware
 347          * isn't caught up. Thus, we need to catch this early return in
 348          * userland and bother the kernel until the hardware really does
 349          * catch up.
 350          *
 351          * E) Hardware might wrap after we test in userland.
 352          *                                                  hw_seq  seq
 353          *                                                      |    |
 354          * ------------------------------------------------------------
 355          * seq - hw_seq = 5.  If we call IRQ_WAIT, it will likely see seq >=
 356          * hw_seq and wait.  However, suppose hw_seq wraps before we make it
 357          * into the kernel.  The kernel sees hw_seq >= seq and waits for 3
 358          * seconds then returns -EBUSY.  This is case C).  We should catch
 359          * this and then return successfully.
 360          *
 361          * F) Hardware might take a long time on a buffer.
 362          * hw_seq seq
 363          *   |    |
 364          * -------------------------------------------------------------------
 365          * seq - hw_seq = 5.  If we call IRQ_WAIT, if sequence 2 through 5
 366          * take too long, it will return -EBUSY.  Batchbuffers in the
 367          * gltestperf demo were seen to take up to 7 seconds.  We should
 368          * catch early -EBUSY return and keep trying.
 369          */
 370
 371         do {
 372                 /* Keep a copy of last_dispatch so that if the wait -EBUSYs
 373                  * because the hardware didn't catch up in 3 seconds, we can
 374                  * see if it at least made progress and retry.
 375                  */
 376                 hw_seq = *bufmgr_fake->last_dispatch;
 377
 378                 /* Catch case C */
 379                 if (seq - hw_seq > 0x40000000)
 380                         return;
 381
 382                 ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT,
 383                                       &iw, sizeof(iw));
 384                 /* Catch case D */
 385                 kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch <
 386                                              -0x40000000);
 387
 388                 /* Catch case E */
 389                 if (ret == -EBUSY
 390                     && (seq - *bufmgr_fake->last_dispatch > 0x40000000))
 391                         ret = 0;
 392
 393                 /* Catch case F: Allow up to 15 seconds chewing on one buffer. */
 394                 if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch))
 395                         busy_count = 0;
 396                 else
 397                         busy_count++;
 398         } while (kernel_lied || ret == -EAGAIN || ret == -EINTR ||
 399                  (ret == -EBUSY && busy_count < 5));
 400
 401         if (ret != 0) {
 402                 drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__,
 403                        __LINE__, strerror(-ret));
 404                 abort();
 405         }
 406         clear_fenced(bufmgr_fake, seq);
 407 }
 408
 409 static int
 410 _fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
 411 {
 412         /* Slight problem with wrap-around:
 413          */
 414         return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence);
 415 }
 416
 417 /**
 418  * Allocate a memory manager block for the buffer.
 419  */
 420 static int
 421 alloc_block(drm_intel_bo *bo)
 422 {
 423         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 424         drm_intel_bufmgr_fake *bufmgr_fake =
 425             (drm_intel_bufmgr_fake *) bo->bufmgr;
 426         struct block *block = (struct block *)calloc(sizeof *block, 1);
 427         unsigned int align_log2 = ffs(bo_fake->alignment) - 1;
 428         unsigned int sz;
 429
 430         if (!block)
 431                 return 1;
 432
 433         sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1);
 434
 435         block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0);
 436         if (!block->mem) {
 437                 free(block);
 438                 return 0;
 439         }
 440
 441         DRMINITLISTHEAD(block);
 442
 443         /* Insert at head or at tail??? */
 444         DRMLISTADDTAIL(block, &bufmgr_fake->lru);
 445
 446         block->virtual = (uint8_t *) bufmgr_fake->virtual +
 447             block->mem->ofs - bufmgr_fake->low_offset;
 448         block->bo = bo;
 449
 450         bo_fake->block = block;
 451
 452         return 1;
 453 }
 454
 455 /* Release the card storage associated with buf:
 456  */
 457 static void
 458 free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block,
 459            int skip_dirty_copy)
 460 {
 461         drm_intel_bo_fake *bo_fake;
 462         DBG("free block %p %08x %d %d\n", block, block->mem->ofs,
 463             block->on_hardware, block->fenced);
 464
 465         if (!block)
 466                 return;
 467
 468         bo_fake = (drm_intel_bo_fake *) block->bo;
 469
 470         if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))
 471                 skip_dirty_copy = 1;
 472
 473         if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) {
 474                 memcpy(bo_fake->backing_store, block->virtual, block->bo->size);
 475                 bo_fake->card_dirty = 0;
 476                 bo_fake->dirty = 1;
 477         }
 478
 479         if (block->on_hardware) {
 480                 block->bo = NULL;
 481         } else if (block->fenced) {
 482                 block->bo = NULL;
 483         } else {
 484                 DBG("    - free immediately\n");
 485                 DRMLISTDEL(block);
 486
 487                 mmFreeMem(block->mem);
 488                 free(block);
 489         }
 490 }
 491
 492 static void
 493 alloc_backing_store(drm_intel_bo *bo)
 494 {
 495         drm_intel_bufmgr_fake *bufmgr_fake =
 496             (drm_intel_bufmgr_fake *) bo->bufmgr;
 497         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 498         assert(!bo_fake->backing_store);
 499         assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));
 500
 501         bo_fake->backing_store = malloc(bo->size);
 502
 503         DBG("alloc_backing - buf %d %p %d\n", bo_fake->id,
 504             bo_fake->backing_store, bo->size);
 505         assert(bo_fake->backing_store);
 506 }
 507
 508 static void
 509 free_backing_store(drm_intel_bo *bo)
 510 {
 511         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 512
 513         if (bo_fake->backing_store) {
 514                 assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));
 515                 free(bo_fake->backing_store);
 516                 bo_fake->backing_store = NULL;
 517         }
 518 }
 519
 520 static void
 521 set_dirty(drm_intel_bo *bo)
 522 {
 523         drm_intel_bufmgr_fake *bufmgr_fake =
 524             (drm_intel_bufmgr_fake *) bo->bufmgr;
 525         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 526
 527         if (bo_fake->flags & BM_NO_BACKING_STORE
 528             && bo_fake->invalidate_cb != NULL)
 529                 bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr);
 530
 531         assert(!(bo_fake->flags & BM_PINNED));
 532
 533         DBG("set_dirty - buf %d\n", bo_fake->id);
 534         bo_fake->dirty = 1;
 535 }
 536
 537 static int
 538 evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence)
 539 {
 540         struct block *block, *tmp;
 541
 542         DBG("%s\n", __FUNCTION__);
 543
 544         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
 545                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
 546
 547                 if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
 548                         continue;
 549
 550                 if (block->fence && max_fence && !FENCE_LTE(block->fence,
 551                                                             max_fence))
 552                         return 0;
 553
 554                 set_dirty(&bo_fake->bo);
 555                 bo_fake->block = NULL;
 556
 557                 free_block(bufmgr_fake, block, 0);
 558                 return 1;
 559         }
 560
 561         return 0;
 562 }
 563
 564 static int
 565 evict_mru(drm_intel_bufmgr_fake *bufmgr_fake)
 566 {
 567         struct block *block, *tmp;
 568
 569         DBG("%s\n", __FUNCTION__);
 570
 571         DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) {
 572                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
 573
 574                 if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
 575                         continue;
 576
 577                 set_dirty(&bo_fake->bo);
 578                 bo_fake->block = NULL;
 579
 580                 free_block(bufmgr_fake, block, 0);
 581                 return 1;
 582         }
 583
 584         return 0;
 585 }
 586
 587 /**
 588  * Removes all objects from the fenced list older than the given fence.
 589  */
 590 static int
 591 clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie)
 592 {
 593         struct block *block, *tmp;
 594         int ret = 0;
 595
 596         bufmgr_fake->last_fence = fence_cookie;
 597         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) {
 598                 assert(block->fenced);
 599
 600                 if (_fence_test(bufmgr_fake, block->fence)) {
 601
 602                         block->fenced = 0;
 603
 604                         if (!block->bo) {
 605                                 DBG("delayed free: offset %x sz %x\n",
 606                                     block->mem->ofs, block->mem->size);
 607                                 DRMLISTDEL(block);
 608                                 mmFreeMem(block->mem);
 609                                 free(block);
 610                         } else {
 611                                 DBG("return to lru: offset %x sz %x\n",
 612                                     block->mem->ofs, block->mem->size);
 613                                 DRMLISTDEL(block);
 614                                 DRMLISTADDTAIL(block, &bufmgr_fake->lru);
 615                         }
 616
 617                         ret = 1;
 618                 } else {
 619                         /* Blocks are ordered by fence, so if one fails, all
 620                          * from here will fail also:
 621                          */
 622                         DBG("fence not passed: offset %x sz %x %d %d \n",
 623                             block->mem->ofs, block->mem->size, block->fence,
 624                             bufmgr_fake->last_fence);
 625                         break;
 626                 }
 627         }
 628
 629         DBG("%s: %d\n", __FUNCTION__, ret);
 630         return ret;
 631 }
 632
 633 static void
 634 fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
 635 {
 636         struct block *block, *tmp;
 637
 638         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
 639                 DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n",
 640                     block, block->mem->size, block->mem->ofs, block->bo, fence);
 641                 block->fence = fence;
 642
 643                 block->on_hardware = 0;
 644                 block->fenced = 1;
 645
 646                 /* Move to tail of pending list here
 647                  */
 648                 DRMLISTDEL(block);
 649                 DRMLISTADDTAIL(block, &bufmgr_fake->fenced);
 650         }
 651
 652         assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
 653 }
 654
 655 static int
 656 evict_and_alloc_block(drm_intel_bo *bo)
 657 {
 658         drm_intel_bufmgr_fake *bufmgr_fake =
 659             (drm_intel_bufmgr_fake *) bo->bufmgr;
 660         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 661
 662         assert(bo_fake->block == NULL);
 663
 664         /* Search for already free memory:
 665          */
 666         if (alloc_block(bo))
 667                 return 1;
 668
 669         /* If we're not thrashing, allow lru eviction to dig deeper into
 670          * recently used textures.  We'll probably be thrashing soon:
 671          */
 672         if (!bufmgr_fake->thrashing) {
 673                 while (evict_lru(bufmgr_fake, 0))
 674                         if (alloc_block(bo))
 675                                 return 1;
 676         }
 677
 678         /* Keep thrashing counter alive?
 679          */
 680         if (bufmgr_fake->thrashing)
 681                 bufmgr_fake->thrashing = 20;
 682
 683         /* Wait on any already pending fences - here we are waiting for any
 684          * freed memory that has been submitted to hardware and fenced to
 685          * become available:
 686          */
 687         while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
 688                 uint32_t fence = bufmgr_fake->fenced.next->fence;
 689                 _fence_wait_internal(bufmgr_fake, fence);
 690
 691                 if (alloc_block(bo))
 692                         return 1;
 693         }
 694
 695         if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) {
 696                 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
 697                         uint32_t fence = bufmgr_fake->fenced.next->fence;
 698                         _fence_wait_internal(bufmgr_fake, fence);
 699                 }
 700
 701                 if (!bufmgr_fake->thrashing) {
 702                         DBG("thrashing\n");
 703                 }
 704                 bufmgr_fake->thrashing = 20;
 705
 706                 if (alloc_block(bo))
 707                         return 1;
 708         }
 709
 710         while (evict_mru(bufmgr_fake))
 711                 if (alloc_block(bo))
 712                         return 1;
 713
 714         DBG("%s 0x%x bytes failed\n", __FUNCTION__, bo->size);
 715
 716         return 0;
 717 }
 718
 719 /***********************************************************************
 720  * Public functions
 721  */
 722
 723 /**
 724  * Wait for hardware idle by emitting a fence and waiting for it.
 725  */
 726 static void
 727 drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake)
 728 {
 729         unsigned int cookie;
 730
 731         cookie = _fence_emit_internal(bufmgr_fake);
 732         _fence_wait_internal(bufmgr_fake, cookie);
 733 }
 734
 735 /**
 736  * Wait for rendering to a buffer to complete.
 737  *
 738  * It is assumed that the bathcbuffer which performed the rendering included
 739  * the necessary flushing.
 740  */
 741 static void
 742 drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo)
 743 {
 744         drm_intel_bufmgr_fake *bufmgr_fake =
 745             (drm_intel_bufmgr_fake *) bo->bufmgr;
 746         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 747
 748         if (bo_fake->block == NULL || !bo_fake->block->fenced)
 749                 return;
 750
 751         _fence_wait_internal(bufmgr_fake, bo_fake->block->fence);
 752 }
 753
 754 static void
 755 drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo)
 756 {
 757         drm_intel_bufmgr_fake *bufmgr_fake =
 758             (drm_intel_bufmgr_fake *) bo->bufmgr;
 759
 760         pthread_mutex_lock(&bufmgr_fake->lock);
 761         drm_intel_fake_bo_wait_rendering_locked(bo);
 762         pthread_mutex_unlock(&bufmgr_fake->lock);
 763 }
 764
 765 /* Specifically ignore texture memory sharing.
 766  *  -- just evict everything
 767  *  -- and wait for idle
 768  */
 769 void
 770 drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr *bufmgr)
 771 {
 772         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 773         struct block *block, *tmp;
 774
 775         pthread_mutex_lock(&bufmgr_fake->lock);
 776
 777         bufmgr_fake->need_fence = 1;
 778         bufmgr_fake->fail = 0;
 779
 780         /* Wait for hardware idle.  We don't know where acceleration has been
 781          * happening, so we'll need to wait anyway before letting anything get
 782          * put on the card again.
 783          */
 784         drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
 785
 786         /* Check that we hadn't released the lock without having fenced the last
 787          * set of buffers.
 788          */
 789         assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
 790         assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
 791
 792         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
 793                 assert(_fence_test(bufmgr_fake, block->fence));
 794                 set_dirty(block->bo);
 795         }
 796
 797         pthread_mutex_unlock(&bufmgr_fake->lock);
 798 }
 799
 800 static drm_intel_bo *
 801 drm_intel_fake_bo_alloc(drm_intel_bufmgr *bufmgr,
 802                         const char *name,
 803                         unsigned long size,
 804                         unsigned int alignment)
 805 {
 806         drm_intel_bufmgr_fake *bufmgr_fake;
 807         drm_intel_bo_fake *bo_fake;
 808
 809         bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 810
 811         assert(size != 0);
 812
 813         bo_fake = calloc(1, sizeof(*bo_fake));
 814         if (!bo_fake)
 815                 return NULL;
 816
 817         bo_fake->bo.size = size;
 818         bo_fake->bo.offset = -1;
 819         bo_fake->bo.virtual = NULL;
 820         bo_fake->bo.bufmgr = bufmgr;
 821         bo_fake->refcount = 1;
 822
 823         /* Alignment must be a power of two */
 824         assert((alignment & (alignment - 1)) == 0);
 825         if (alignment == 0)
 826                 alignment = 1;
 827         bo_fake->alignment = alignment;
 828         bo_fake->id = ++bufmgr_fake->buf_nr;
 829         bo_fake->name = name;
 830         bo_fake->flags = 0;
 831         bo_fake->is_static = 0;
 832
 833         DBG("drm_bo_alloc: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name,
 834             bo_fake->bo.size / 1024);
 835
 836         return &bo_fake->bo;
 837 }
 838
 839 static drm_intel_bo *
 840 drm_intel_fake_bo_alloc_tiled(drm_intel_bufmgr * bufmgr,
 841                               const char *name,
 842                               int x, int y, int cpp,
 843                               uint32_t *tiling_mode,
 844                               unsigned long *pitch,
 845                               unsigned long flags)
 846 {
 847         unsigned long stride, aligned_y;
 848
 849         /* No runtime tiling support for fake. */
 850         *tiling_mode = I915_TILING_NONE;
 851
 852         /* Align it for being a render target.  Shouldn't need anything else. */
 853         stride = x * cpp;
 854         stride = ROUND_UP_TO(stride, 64);
 855
 856         /* 965 subspan loading alignment */
 857         aligned_y = ALIGN(y, 2);
 858
 859         *pitch = stride;
 860
 861         return drm_intel_fake_bo_alloc(bufmgr, name, stride * aligned_y,
 862                                        4096);
 863 }
 864
 865 drm_intel_bo *
 866 drm_intel_bo_fake_alloc_static(drm_intel_bufmgr *bufmgr,
 867                                const char *name,
 868                                unsigned long offset,
 869                                unsigned long size, void *virtual)
 870 {
 871         drm_intel_bufmgr_fake *bufmgr_fake;
 872         drm_intel_bo_fake *bo_fake;
 873
 874         bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
 875
 876         assert(size != 0);
 877
 878         bo_fake = calloc(1, sizeof(*bo_fake));
 879         if (!bo_fake)
 880                 return NULL;
 881
 882         bo_fake->bo.size = size;
 883         bo_fake->bo.offset = offset;
 884         bo_fake->bo.virtual = virtual;
 885         bo_fake->bo.bufmgr = bufmgr;
 886         bo_fake->refcount = 1;
 887         bo_fake->id = ++bufmgr_fake->buf_nr;
 888         bo_fake->name = name;
 889         bo_fake->flags = BM_PINNED;
 890         bo_fake->is_static = 1;
 891
 892         DBG("drm_bo_alloc_static: (buf %d: %s, %d kb)\n", bo_fake->id,
 893             bo_fake->name, bo_fake->bo.size / 1024);
 894
 895         return &bo_fake->bo;
 896 }
 897
 898 static void
 899 drm_intel_fake_bo_reference(drm_intel_bo *bo)
 900 {
 901         drm_intel_bufmgr_fake *bufmgr_fake =
 902             (drm_intel_bufmgr_fake *) bo->bufmgr;
 903         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 904
 905         pthread_mutex_lock(&bufmgr_fake->lock);
 906         bo_fake->refcount++;
 907         pthread_mutex_unlock(&bufmgr_fake->lock);
 908 }
 909
 910 static void
 911 drm_intel_fake_bo_reference_locked(drm_intel_bo *bo)
 912 {
 913         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 914
 915         bo_fake->refcount++;
 916 }
 917
 918 static void
 919 drm_intel_fake_bo_unreference_locked(drm_intel_bo *bo)
 920 {
 921         drm_intel_bufmgr_fake *bufmgr_fake =
 922             (drm_intel_bufmgr_fake *) bo->bufmgr;
 923         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 924         int i;
 925
 926         if (--bo_fake->refcount == 0) {
 927                 assert(bo_fake->map_count == 0);
 928                 /* No remaining references, so free it */
 929                 if (bo_fake->block)
 930                         free_block(bufmgr_fake, bo_fake->block, 1);
 931                 free_backing_store(bo);
 932
 933                 for (i = 0; i < bo_fake->nr_relocs; i++)
 934                         drm_intel_fake_bo_unreference_locked(bo_fake->relocs[i].
 935                                                              target_buf);
 936
 937                 DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id,
 938                     bo_fake->name);
 939
 940                 free(bo_fake->relocs);
 941                 free(bo);
 942         }
 943 }
 944
 945 static void
 946 drm_intel_fake_bo_unreference(drm_intel_bo *bo)
 947 {
 948         drm_intel_bufmgr_fake *bufmgr_fake =
 949             (drm_intel_bufmgr_fake *) bo->bufmgr;
 950
 951         pthread_mutex_lock(&bufmgr_fake->lock);
 952         drm_intel_fake_bo_unreference_locked(bo);
 953         pthread_mutex_unlock(&bufmgr_fake->lock);
 954 }
 955
 956 /**
 957  * Set the buffer as not requiring backing store, and instead get the callback
 958  * invoked whenever it would be set dirty.
 959  */
 960 void
 961 drm_intel_bo_fake_disable_backing_store(drm_intel_bo *bo,
 962                                         void (*invalidate_cb) (drm_intel_bo *bo,
 963                                                                void *ptr),
 964                                         void *ptr)
 965 {
 966         drm_intel_bufmgr_fake *bufmgr_fake =
 967             (drm_intel_bufmgr_fake *) bo->bufmgr;
 968         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
 969
 970         pthread_mutex_lock(&bufmgr_fake->lock);
 971
 972         if (bo_fake->backing_store)
 973                 free_backing_store(bo);
 974
 975         bo_fake->flags |= BM_NO_BACKING_STORE;
 976
 977         DBG("disable_backing_store set buf %d dirty\n", bo_fake->id);
 978         bo_fake->dirty = 1;
 979         bo_fake->invalidate_cb = invalidate_cb;
 980         bo_fake->invalidate_ptr = ptr;
 981
 982         /* Note that it is invalid right from the start.  Also note
 983          * invalidate_cb is called with the bufmgr locked, so cannot
 984          * itself make bufmgr calls.
 985          */
 986         if (invalidate_cb != NULL)
 987                 invalidate_cb(bo, ptr);
 988
 989         pthread_mutex_unlock(&bufmgr_fake->lock);
 990 }
 991
 992 /**
 993  * Map a buffer into bo->virtual, allocating either card memory space (If
 994  * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary.
 995  */
 996 static int
 997  drm_intel_fake_bo_map_locked(drm_intel_bo *bo, int write_enable)
 998 {
 999         drm_intel_bufmgr_fake *bufmgr_fake =
1000             (drm_intel_bufmgr_fake *) bo->bufmgr;
1001         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1002
1003         /* Static buffers are always mapped. */
1004         if (bo_fake->is_static) {
1005                 if (bo_fake->card_dirty) {
1006                         drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1007                         bo_fake->card_dirty = 0;
1008                 }
1009                 return 0;
1010         }
1011
1012         /* Allow recursive mapping.  Mesa may recursively map buffers with
1013          * nested display loops, and it is used internally in bufmgr_fake
1014          * for relocation.
1015          */
1016         if (bo_fake->map_count++ != 0)
1017                 return 0;
1018
1019         {
1020                 DBG("drm_bo_map: (buf %d: %s, %d kb)\n", bo_fake->id,
1021                     bo_fake->name, bo_fake->bo.size / 1024);
1022
1023                 if (bo->virtual != NULL) {
1024                         drmMsg("%s: already mapped\n", __FUNCTION__);
1025                         abort();
1026                 } else if (bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)) {
1027
1028                         if (!bo_fake->block && !evict_and_alloc_block(bo)) {
1029                                 DBG("%s: alloc failed\n", __FUNCTION__);
1030                                 bufmgr_fake->fail = 1;
1031                                 return 1;
1032                         } else {
1033                                 assert(bo_fake->block);
1034                                 bo_fake->dirty = 0;
1035
1036                                 if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) &&
1037                                     bo_fake->block->fenced) {
1038                                         drm_intel_fake_bo_wait_rendering_locked
1039                                             (bo);
1040                                 }
1041
1042                                 bo->virtual = bo_fake->block->virtual;
1043                         }
1044                 } else {
1045                         if (write_enable)
1046                                 set_dirty(bo);
1047
1048                         if (bo_fake->backing_store == 0)
1049                                 alloc_backing_store(bo);
1050
1051                         if ((bo_fake->card_dirty == 1) && bo_fake->block) {
1052                                 if (bo_fake->block->fenced)
1053                                         drm_intel_fake_bo_wait_rendering_locked
1054                                             (bo);
1055
1056                                 memcpy(bo_fake->backing_store,
1057                                        bo_fake->block->virtual,
1058                                        bo_fake->block->bo->size);
1059                                 bo_fake->card_dirty = 0;
1060                         }
1061
1062                         bo->virtual = bo_fake->backing_store;
1063                 }
1064         }
1065
1066         return 0;
1067 }
1068
1069 static int
1070  drm_intel_fake_bo_map(drm_intel_bo *bo, int write_enable)
1071 {
1072         drm_intel_bufmgr_fake *bufmgr_fake =
1073             (drm_intel_bufmgr_fake *) bo->bufmgr;
1074         int ret;
1075
1076         pthread_mutex_lock(&bufmgr_fake->lock);
1077         ret = drm_intel_fake_bo_map_locked(bo, write_enable);
1078         pthread_mutex_unlock(&bufmgr_fake->lock);
1079
1080         return ret;
1081 }
1082
1083 static int
1084  drm_intel_fake_bo_unmap_locked(drm_intel_bo *bo)
1085 {
1086         drm_intel_bufmgr_fake *bufmgr_fake =
1087             (drm_intel_bufmgr_fake *) bo->bufmgr;
1088         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1089
1090         /* Static buffers are always mapped. */
1091         if (bo_fake->is_static)
1092                 return 0;
1093
1094         assert(bo_fake->map_count != 0);
1095         if (--bo_fake->map_count != 0)
1096                 return 0;
1097
1098         DBG("drm_bo_unmap: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name,
1099             bo_fake->bo.size / 1024);
1100
1101         bo->virtual = NULL;
1102
1103         return 0;
1104 }
1105
1106 static int drm_intel_fake_bo_unmap(drm_intel_bo *bo)
1107 {
1108         drm_intel_bufmgr_fake *bufmgr_fake =
1109             (drm_intel_bufmgr_fake *) bo->bufmgr;
1110         int ret;
1111
1112         pthread_mutex_lock(&bufmgr_fake->lock);
1113         ret = drm_intel_fake_bo_unmap_locked(bo);
1114         pthread_mutex_unlock(&bufmgr_fake->lock);
1115
1116         return ret;
1117 }
1118
1119 static void
1120  drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake *bufmgr_fake)
1121 {
1122         struct block *block, *tmp;
1123
1124         bufmgr_fake->performed_rendering = 0;
1125         /* okay for ever BO that is on the HW kick it off.
1126            seriously not afraid of the POLICE right now */
1127         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
1128                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
1129
1130                 block->on_hardware = 0;
1131                 free_block(bufmgr_fake, block, 0);
1132                 bo_fake->block = NULL;
1133                 bo_fake->validated = 0;
1134                 if (!(bo_fake->flags & BM_NO_BACKING_STORE))
1135                         bo_fake->dirty = 1;
1136         }
1137
1138 }
1139
1140 static int
1141  drm_intel_fake_bo_validate(drm_intel_bo *bo)
1142 {
1143         drm_intel_bufmgr_fake *bufmgr_fake;
1144         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1145
1146         bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr;
1147
1148         DBG("drm_bo_validate: (buf %d: %s, %d kb)\n", bo_fake->id,
1149             bo_fake->name, bo_fake->bo.size / 1024);
1150
1151         /* Sanity check: Buffers should be unmapped before being validated.
1152          * This is not so much of a problem for bufmgr_fake, but TTM refuses,
1153          * and the problem is harder to debug there.
1154          */
1155         assert(bo_fake->map_count == 0);
1156
1157         if (bo_fake->is_static) {
1158                 /* Add it to the needs-fence list */
1159                 bufmgr_fake->need_fence = 1;
1160                 return 0;
1161         }
1162
1163         /* Allocate the card memory */
1164         if (!bo_fake->block && !evict_and_alloc_block(bo)) {
1165                 bufmgr_fake->fail = 1;
1166                 DBG("Failed to validate buf %d:%s\n", bo_fake->id,
1167                     bo_fake->name);
1168                 return -1;
1169         }
1170
1171         assert(bo_fake->block);
1172         assert(bo_fake->block->bo == &bo_fake->bo);
1173
1174         bo->offset = bo_fake->block->mem->ofs;
1175
1176         /* Upload the buffer contents if necessary */
1177         if (bo_fake->dirty) {
1178                 DBG("Upload dirty buf %d:%s, sz %d offset 0x%x\n", bo_fake->id,
1179                     bo_fake->name, bo->size, bo_fake->block->mem->ofs);
1180
1181                 assert(!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)));
1182
1183                 /* Actually, should be able to just wait for a fence on the
1184                  * mmory, hich we would be tracking when we free it.  Waiting
1185                  * for idle is a sufficiently large hammer for now.
1186                  */
1187                 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1188
1189                 /* we may never have mapped this BO so it might not have any
1190                  * backing store if this happens it should be rare, but 0 the
1191                  * card memory in any case */
1192                 if (bo_fake->backing_store)
1193                         memcpy(bo_fake->block->virtual, bo_fake->backing_store,
1194                                bo->size);
1195                 else
1196                         memset(bo_fake->block->virtual, 0, bo->size);
1197
1198                 bo_fake->dirty = 0;
1199         }
1200
1201         bo_fake->block->fenced = 0;
1202         bo_fake->block->on_hardware = 1;
1203         DRMLISTDEL(bo_fake->block);
1204         DRMLISTADDTAIL(bo_fake->block, &bufmgr_fake->on_hardware);
1205
1206         bo_fake->validated = 1;
1207         bufmgr_fake->need_fence = 1;
1208
1209         return 0;
1210 }
1211
1212 static void
1213 drm_intel_fake_fence_validated(drm_intel_bufmgr *bufmgr)
1214 {
1215         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1216         unsigned int cookie;
1217
1218         cookie = _fence_emit_internal(bufmgr_fake);
1219         fence_blocks(bufmgr_fake, cookie);
1220
1221         DBG("drm_fence_validated: 0x%08x cookie\n", cookie);
1222 }
1223
1224 static void
1225 drm_intel_fake_destroy(drm_intel_bufmgr *bufmgr)
1226 {
1227         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1228
1229         pthread_mutex_destroy(&bufmgr_fake->lock);
1230         mmDestroy(bufmgr_fake->heap);
1231         free(bufmgr);
1232 }
1233
1234 static int
1235 drm_intel_fake_emit_reloc(drm_intel_bo *bo, uint32_t offset,
1236                           drm_intel_bo *target_bo, uint32_t target_offset,
1237                           uint32_t read_domains, uint32_t write_domain)
1238 {
1239         drm_intel_bufmgr_fake *bufmgr_fake =
1240             (drm_intel_bufmgr_fake *) bo->bufmgr;
1241         struct fake_buffer_reloc *r;
1242         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1243         drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *) target_bo;
1244         int i;
1245
1246         pthread_mutex_lock(&bufmgr_fake->lock);
1247
1248         assert(bo);
1249         assert(target_bo);
1250
1251         if (bo_fake->relocs == NULL) {
1252                 bo_fake->relocs =
1253                     malloc(sizeof(struct fake_buffer_reloc) * MAX_RELOCS);
1254         }
1255
1256         r = &bo_fake->relocs[bo_fake->nr_relocs++];
1257
1258         assert(bo_fake->nr_relocs <= MAX_RELOCS);
1259
1260         drm_intel_fake_bo_reference_locked(target_bo);
1261
1262         if (!target_fake->is_static) {
1263                 bo_fake->child_size +=
1264                     ALIGN(target_bo->size, target_fake->alignment);
1265                 bo_fake->child_size += target_fake->child_size;
1266         }
1267         r->target_buf = target_bo;
1268         r->offset = offset;
1269         r->last_target_offset = target_bo->offset;
1270         r->delta = target_offset;
1271         r->read_domains = read_domains;
1272         r->write_domain = write_domain;
1273
1274         if (bufmgr_fake->debug) {
1275                 /* Check that a conflicting relocation hasn't already been
1276                  * emitted.
1277                  */
1278                 for (i = 0; i < bo_fake->nr_relocs - 1; i++) {
1279                         struct fake_buffer_reloc *r2 = &bo_fake->relocs[i];
1280
1281                         assert(r->offset != r2->offset);
1282                 }
1283         }
1284
1285         pthread_mutex_unlock(&bufmgr_fake->lock);
1286
1287         return 0;
1288 }
1289
1290 /**
1291  * Incorporates the validation flags associated with each relocation into
1292  * the combined validation flags for the buffer on this batchbuffer submission.
1293  */
1294 static void
1295 drm_intel_fake_calculate_domains(drm_intel_bo *bo)
1296 {
1297         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1298         int i;
1299
1300         for (i = 0; i < bo_fake->nr_relocs; i++) {
1301                 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1302                 drm_intel_bo_fake *target_fake =
1303                     (drm_intel_bo_fake *) r->target_buf;
1304
1305                 /* Do the same for the tree of buffers we depend on */
1306                 drm_intel_fake_calculate_domains(r->target_buf);
1307
1308                 target_fake->read_domains |= r->read_domains;
1309                 target_fake->write_domain |= r->write_domain;
1310         }
1311 }
1312
1313 static int
1314 drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo *bo)
1315 {
1316         drm_intel_bufmgr_fake *bufmgr_fake =
1317             (drm_intel_bufmgr_fake *) bo->bufmgr;
1318         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1319         int i, ret;
1320
1321         assert(bo_fake->map_count == 0);
1322
1323         for (i = 0; i < bo_fake->nr_relocs; i++) {
1324                 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1325                 drm_intel_bo_fake *target_fake =
1326                     (drm_intel_bo_fake *) r->target_buf;
1327                 uint32_t reloc_data;
1328
1329                 /* Validate the target buffer if that hasn't been done. */
1330                 if (!target_fake->validated) {
1331                         ret =
1332                             drm_intel_fake_reloc_and_validate_buffer(r->target_buf);
1333                         if (ret != 0) {
1334                                 if (bo->virtual != NULL)
1335                                         drm_intel_fake_bo_unmap_locked(bo);
1336                                 return ret;
1337                         }
1338                 }
1339
1340                 /* Calculate the value of the relocation entry. */
1341                 if (r->target_buf->offset != r->last_target_offset) {
1342                         reloc_data = r->target_buf->offset + r->delta;
1343
1344                         if (bo->virtual == NULL)
1345                                 drm_intel_fake_bo_map_locked(bo, 1);
1346
1347                         *(uint32_t *) ((uint8_t *) bo->virtual + r->offset) =
1348                             reloc_data;
1349
1350                         r->last_target_offset = r->target_buf->offset;
1351                 }
1352         }
1353
1354         if (bo->virtual != NULL)
1355                 drm_intel_fake_bo_unmap_locked(bo);
1356
1357         if (bo_fake->write_domain != 0) {
1358                 if (!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED))) {
1359                         if (bo_fake->backing_store == 0)
1360                                 alloc_backing_store(bo);
1361                 }
1362                 bo_fake->card_dirty = 1;
1363                 bufmgr_fake->performed_rendering = 1;
1364         }
1365
1366         return drm_intel_fake_bo_validate(bo);
1367 }
1368
1369 static void
1370 drm_intel_bo_fake_post_submit(drm_intel_bo *bo)
1371 {
1372         drm_intel_bufmgr_fake *bufmgr_fake =
1373             (drm_intel_bufmgr_fake *) bo->bufmgr;
1374         drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1375         int i;
1376
1377         for (i = 0; i < bo_fake->nr_relocs; i++) {
1378                 struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1379                 drm_intel_bo_fake *target_fake =
1380                     (drm_intel_bo_fake *) r->target_buf;
1381
1382                 if (target_fake->validated)
1383                         drm_intel_bo_fake_post_submit(r->target_buf);
1384
1385                 DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n",
1386                     bo_fake->name, (uint32_t) bo->offset, r->offset,
1387                     target_fake->name, (uint32_t) r->target_buf->offset,
1388                     r->delta);
1389         }
1390
1391         assert(bo_fake->map_count == 0);
1392         bo_fake->validated = 0;
1393         bo_fake->read_domains = 0;
1394         bo_fake->write_domain = 0;
1395 }
1396
1397 void
1398 drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr *bufmgr,
1399                                              int (*exec) (drm_intel_bo *bo,
1400                                                           unsigned int used,
1401                                                           void *priv),
1402                                              void *priv)
1403 {
1404         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1405
1406         bufmgr_fake->exec = exec;
1407         bufmgr_fake->exec_priv = priv;
1408 }
1409
1410 static int
1411 drm_intel_fake_bo_exec(drm_intel_bo *bo, int used,
1412                        drm_clip_rect_t * cliprects, int num_cliprects, int DR4)
1413 {
1414         drm_intel_bufmgr_fake *bufmgr_fake =
1415             (drm_intel_bufmgr_fake *) bo->bufmgr;
1416         drm_intel_bo_fake *batch_fake = (drm_intel_bo_fake *) bo;
1417         struct drm_i915_batchbuffer batch;
1418         int ret;
1419         int retry_count = 0;
1420
1421         pthread_mutex_lock(&bufmgr_fake->lock);
1422
1423         bufmgr_fake->performed_rendering = 0;
1424
1425         drm_intel_fake_calculate_domains(bo);
1426
1427         batch_fake->read_domains = I915_GEM_DOMAIN_COMMAND;
1428
1429         /* we've ran out of RAM so blow the whole lot away and retry */
1430 restart:
1431         ret = drm_intel_fake_reloc_and_validate_buffer(bo);
1432         if (bufmgr_fake->fail == 1) {
1433                 if (retry_count == 0) {
1434                         retry_count++;
1435                         drm_intel_fake_kick_all_locked(bufmgr_fake);
1436                         bufmgr_fake->fail = 0;
1437                         goto restart;
1438                 } else          /* dump out the memory here */
1439                         mmDumpMemInfo(bufmgr_fake->heap);
1440         }
1441
1442         assert(ret == 0);
1443
1444         if (bufmgr_fake->exec != NULL) {
1445                 int ret = bufmgr_fake->exec(bo, used, bufmgr_fake->exec_priv);
1446                 if (ret != 0) {
1447                         pthread_mutex_unlock(&bufmgr_fake->lock);
1448                         return ret;
1449                 }
1450         } else {
1451                 batch.start = bo->offset;
1452                 batch.used = used;
1453                 batch.cliprects = cliprects;
1454                 batch.num_cliprects = num_cliprects;
1455                 batch.DR1 = 0;
1456                 batch.DR4 = DR4;
1457
1458                 if (drmCommandWrite
1459                     (bufmgr_fake->fd, DRM_I915_BATCHBUFFER, &batch,
1460                      sizeof(batch))) {
1461                         drmMsg("DRM_I915_BATCHBUFFER: %d\n", -errno);
1462                         pthread_mutex_unlock(&bufmgr_fake->lock);
1463                         return -errno;
1464                 }
1465         }
1466
1467         drm_intel_fake_fence_validated(bo->bufmgr);
1468
1469         drm_intel_bo_fake_post_submit(bo);
1470
1471         pthread_mutex_unlock(&bufmgr_fake->lock);
1472
1473         return 0;
1474 }
1475
1476 /**
1477  * Return an error if the list of BOs will exceed the aperture size.
1478  *
1479  * This is a rough guess and likely to fail, as during the validate sequence we
1480  * may place a buffer in an inopportune spot early on and then fail to fit
1481  * a set smaller than the aperture.
1482  */
1483 static int
1484 drm_intel_fake_check_aperture_space(drm_intel_bo ** bo_array, int count)
1485 {
1486         drm_intel_bufmgr_fake *bufmgr_fake =
1487             (drm_intel_bufmgr_fake *) bo_array[0]->bufmgr;
1488         unsigned int sz = 0;
1489         int i;
1490
1491         for (i = 0; i < count; i++) {
1492                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo_array[i];
1493
1494                 if (bo_fake == NULL)
1495                         continue;
1496
1497                 if (!bo_fake->is_static)
1498                         sz += ALIGN(bo_array[i]->size, bo_fake->alignment);
1499                 sz += bo_fake->child_size;
1500         }
1501
1502         if (sz > bufmgr_fake->size) {
1503                 DBG("check_space: overflowed bufmgr size, %dkb vs %dkb\n",
1504                     sz / 1024, bufmgr_fake->size / 1024);
1505                 return -1;
1506         }
1507
1508         DBG("drm_check_space: sz %dkb vs bufgr %dkb\n", sz / 1024,
1509             bufmgr_fake->size / 1024);
1510         return 0;
1511 }
1512
1513 /**
1514  * Evicts all buffers, waiting for fences to pass and copying contents out
1515  * as necessary.
1516  *
1517  * Used by the X Server on LeaveVT, when the card memory is no longer our
1518  * own.
1519  */
1520 void drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr *bufmgr)
1521 {
1522         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1523         struct block *block, *tmp;
1524
1525         pthread_mutex_lock(&bufmgr_fake->lock);
1526
1527         bufmgr_fake->need_fence = 1;
1528         bufmgr_fake->fail = 0;
1529
1530         /* Wait for hardware idle.  We don't know where acceleration has been
1531          * happening, so we'll need to wait anyway before letting anything get
1532          * put on the card again.
1533          */
1534         drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1535
1536         /* Check that we hadn't released the lock without having fenced the last
1537          * set of buffers.
1538          */
1539         assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
1540         assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
1541
1542         DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
1543                 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
1544                 /* Releases the memory, and memcpys dirty contents out if
1545                  * necessary.
1546                  */
1547                 free_block(bufmgr_fake, block, 0);
1548                 bo_fake->block = NULL;
1549         }
1550
1551         pthread_mutex_unlock(&bufmgr_fake->lock);
1552 }
1553
1554 void drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr *bufmgr,
1555                                              volatile unsigned int
1556                                              *last_dispatch)
1557 {
1558         drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1559
1560         bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1561 }
1562
1563 drm_intel_bufmgr *drm_intel_bufmgr_fake_init(int fd,
1564                                              unsigned long low_offset,
1565                                              void *low_virtual,
1566                                              unsigned long size,
1567                                              volatile unsigned int
1568                                              *last_dispatch)
1569 {
1570         drm_intel_bufmgr_fake *bufmgr_fake;
1571
1572         bufmgr_fake = calloc(1, sizeof(*bufmgr_fake));
1573
1574         if (pthread_mutex_init(&bufmgr_fake->lock, NULL) != 0) {
1575                 free(bufmgr_fake);
1576                 return NULL;
1577         }
1578
1579         /* Initialize allocator */
1580         DRMINITLISTHEAD(&bufmgr_fake->fenced);
1581         DRMINITLISTHEAD(&bufmgr_fake->on_hardware);
1582         DRMINITLISTHEAD(&bufmgr_fake->lru);
1583
1584         bufmgr_fake->low_offset = low_offset;
1585         bufmgr_fake->virtual = low_virtual;
1586         bufmgr_fake->size = size;
1587         bufmgr_fake->heap = mmInit(low_offset, size);
1588
1589         /* Hook in methods */
1590         bufmgr_fake->bufmgr.bo_alloc = drm_intel_fake_bo_alloc;
1591         bufmgr_fake->bufmgr.bo_alloc_for_render = drm_intel_fake_bo_alloc;
1592         bufmgr_fake->bufmgr.bo_alloc_tiled = drm_intel_fake_bo_alloc_tiled;
1593         bufmgr_fake->bufmgr.bo_reference = drm_intel_fake_bo_reference;
1594         bufmgr_fake->bufmgr.bo_unreference = drm_intel_fake_bo_unreference;
1595         bufmgr_fake->bufmgr.bo_map = drm_intel_fake_bo_map;
1596         bufmgr_fake->bufmgr.bo_unmap = drm_intel_fake_bo_unmap;
1597         bufmgr_fake->bufmgr.bo_wait_rendering =
1598             drm_intel_fake_bo_wait_rendering;
1599         bufmgr_fake->bufmgr.bo_emit_reloc = drm_intel_fake_emit_reloc;
1600         bufmgr_fake->bufmgr.destroy = drm_intel_fake_destroy;
1601         bufmgr_fake->bufmgr.bo_exec = drm_intel_fake_bo_exec;
1602         bufmgr_fake->bufmgr.check_aperture_space =
1603             drm_intel_fake_check_aperture_space;
1604         bufmgr_fake->bufmgr.debug = 0;
1605
1606         bufmgr_fake->fd = fd;
1607         bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1608
1609         return &bufmgr_fake->bufmgr;
1610 }