src/wsbm_slabpool.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
   4  * All Rights Reserved.
   5  * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the
  10  * "Software"), to deal in the Software without restriction, including
  11  * without limitation the rights to use, copy, modify, merge, publish,
  12  * distribute, sub license, and/or sell copies of the Software, and to
  13  * permit persons to whom the Software is furnished to do so, subject to
  14  * the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the
  17  * next paragraph) shall be included in all copies or substantial portions
  18  * of the Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  23  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  24  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  25  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  26  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  27  *
  28  **************************************************************************/
  29 /*
  30  * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  31  */
  32
  33 #ifdef HAVE_CONFIG_H
  34 #include "config.h"
  35 #endif
  36
  37 #include <ttm/ttm_placement_user.h>
  38 #include <stdint.h>
  39 #include <sys/time.h>
  40 #include <errno.h>
  41 #include <unistd.h>
  42 #include <assert.h>
  43 #include <string.h>
  44 #include <sys/mman.h>
  45 #include <xf86drm.h>
  46 #include "wsbm_pool.h"
  47 #include "wsbm_fencemgr.h"
  48 #include "wsbm_priv.h"
  49 #include "wsbm_manager.h"
  50
  51 #define WSBM_SLABPOOL_ALLOC_RETRIES 100
  52 #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret)                   \
  53         do {                                                            \
  54                 (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
  55         } while ((_ret) == -EAGAIN || (_ret) == -ERESTART);             \
  56
  57 #define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret)               \
  58         do {                                                            \
  59                 (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
  60         } while ((_ret) == -EAGAIN || (_ret) == -ERESTART);             \
  61
  62
  63 #ifdef DEBUG_FENCESIGNALED
  64 static int createbuffer = 0;
  65 static int fencesignaled = 0;
  66 #endif
  67
  68 struct _WsbmSlab;
  69
  70 struct _WsbmSlabBuffer
  71 {
  72     struct _WsbmKernelBuf kBuf;
  73     struct _WsbmBufStorage storage;
  74     struct _WsbmCond event;
  75
  76     /*
  77      * Remains constant after creation.
  78      */
  79
  80     int isSlabBuffer;
  81     struct _WsbmSlab *parent;
  82     uint32_t start;
  83     void *virtual;
  84     unsigned long requestedSize;
  85     uint64_t mapHandle;
  86
  87     /*
  88      * Protected by struct _WsbmSlabSizeHeader::mutex
  89      */
  90
  91     struct _WsbmListHead head;
  92
  93     /*
  94      * Protected by this::mutex
  95      */
  96
  97     struct _WsbmFenceObject *fence;
  98     uint32_t fenceType;
  99     struct _WsbmAtomic writers;        /* (Only upping) */
 100     int unFenced;
 101 };
 102
 103 struct _WsbmSlabPool;
 104 struct _WsbmSlabKernelBO
 105 {
 106
 107     /*
 108      * Constant at creation
 109      */
 110
 111     struct _WsbmKernelBuf kBuf;
 112     uint32_t pageAlignment;
 113     void *virtual;
 114     unsigned long actualSize;
 115     uint64_t mapHandle;
 116
 117     /*
 118      * Protected by struct _WsbmSlabCache::mutex
 119      */
 120
 121     struct _WsbmSlabPool *slabPool;
 122     uint32_t proposedPlacement;
 123     struct _WsbmListHead timeoutHead;
 124     struct _WsbmListHead head;
 125     struct timeval timeFreed;
 126 };
 127
 128 struct _WsbmSlab
 129 {
 130     struct _WsbmListHead head;
 131     struct _WsbmListHead freeBuffers;
 132     uint32_t numBuffers;
 133     uint32_t numFree;
 134     struct _WsbmSlabBuffer *buffers;
 135     struct _WsbmSlabSizeHeader *header;
 136     struct _WsbmSlabKernelBO *kbo;
 137 };
 138
 139 struct _WsbmSlabSizeHeader
 140 {
 141     /*
 142      * Constant at creation.
 143      */
 144     struct _WsbmSlabPool *slabPool;
 145     uint32_t bufSize;
 146
 147     /*
 148      * Protected by this::mutex
 149      */
 150
 151     struct _WsbmListHead slabs;
 152     struct _WsbmListHead freeSlabs;
 153     struct _WsbmListHead delayedBuffers;
 154     uint32_t numDelayed;
 155     struct _WsbmMutex mutex;
 156 };
 157
 158 struct _WsbmSlabCache
 159 {
 160     struct timeval slabTimeout;
 161     struct timeval checkInterval;
 162     struct timeval nextCheck;
 163     struct _WsbmListHead timeoutList;
 164     struct _WsbmListHead unCached;
 165     struct _WsbmListHead cached;
 166     struct _WsbmMutex mutex;
 167 };
 168
 169 struct _WsbmSlabPool
 170 {
 171     struct _WsbmBufferPool pool;
 172
 173     /*
 174      * The data of this structure remains constant after
 175      * initialization and thus needs no mutex protection.
 176      */
 177
 178     unsigned int devOffset;
 179     struct _WsbmSlabCache *cache;
 180     uint32_t proposedPlacement;
 181     uint32_t validMask;
 182     uint32_t *bucketSizes;
 183     uint32_t numBuckets;
 184     uint32_t pageSize;
 185     int pageAlignment;
 186     int maxSlabSize;
 187     int desiredNumBuffers;
 188     struct _WsbmSlabSizeHeader *headers;
 189 };
 190
 191 static inline struct _WsbmSlabPool *
 192 slabPoolFromPool(struct _WsbmBufferPool *pool)
 193 {
 194     return containerOf(pool, struct _WsbmSlabPool, pool);
 195 }
 196
 197 static inline struct _WsbmSlabPool *
 198 slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf)
 199 {
 200     return slabPoolFromPool(sBuf->storage.pool);
 201 }
 202
 203 static inline struct _WsbmSlabBuffer *
 204 slabBuffer(struct _WsbmBufStorage *buf)
 205 {
 206     return containerOf(buf, struct _WsbmSlabBuffer, storage);
 207 }
 208
 209 /*
 210  * FIXME: Perhaps arrange timeout slabs in size buckets for fast
 211  * retreival??
 212  */
 213
 214 static inline int
 215 wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2)
 216 {
 217     return ((arg1->tv_sec > arg2->tv_sec) ||
 218             ((arg1->tv_sec == arg2->tv_sec) &&
 219              (arg1->tv_usec > arg2->tv_usec)));
 220 }
 221
 222 static inline void
 223 wsbmTimeAdd(struct timeval *arg, struct timeval *add)
 224 {
 225     unsigned int sec;
 226
 227     arg->tv_sec += add->tv_sec;
 228     arg->tv_usec += add->tv_usec;
 229     sec = arg->tv_usec / 1000000;
 230     arg->tv_sec += sec;
 231     arg->tv_usec -= sec * 1000000;
 232 }
 233
 234 static void
 235 wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo)
 236 {
 237     struct ttm_pl_reference_req arg;
 238     struct _WsbmSlabPool *slabPool;
 239
 240     if (!kbo)
 241         return;
 242
 243     slabPool = kbo->slabPool;
 244     arg.handle = kbo->kBuf.handle;
 245     (void)munmap(kbo->virtual, kbo->actualSize);
 246     (void)drmCommandWrite(slabPool->pool.fd,
 247                           slabPool->devOffset + TTM_PL_UNREF, &arg,
 248                           sizeof(arg));
 249     free(kbo);
 250 }
 251
 252 static void
 253 wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time)
 254 {
 255     struct _WsbmListHead *list, *next;
 256     struct _WsbmSlabKernelBO *kbo;
 257
 258     if (!wsbmTimeAfterEq(time, &cache->nextCheck))
 259         return;
 260
 261     WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) {
 262         kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead);
 263
 264         if (!wsbmTimeAfterEq(time, &kbo->timeFreed))
 265             break;
 266
 267         WSBMLISTDELINIT(&kbo->timeoutHead);
 268         WSBMLISTDELINIT(&kbo->head);
 269         wsbmFreeKernelBO(kbo);
 270     }
 271
 272     cache->nextCheck = *time;
 273     wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval);
 274 }
 275
 276 /*
 277  * Add a _SlabKernelBO to the free slab manager.
 278  * This means that it is available for reuse, but if it's not
 279  * reused in a while, it will be freed.
 280  */
 281
 282 static void
 283 wsbmSetKernelBOFree(struct _WsbmSlabCache *cache,
 284                     struct _WsbmSlabKernelBO *kbo)
 285 {
 286     struct timeval time;
 287     struct timeval timeFreed;
 288
 289     gettimeofday(&time, NULL);
 290     timeFreed = time;
 291     WSBM_MUTEX_LOCK(&cache->mutex);
 292     wsbmTimeAdd(&timeFreed, &cache->slabTimeout);
 293     kbo->timeFreed = timeFreed;
 294
 295     if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED)
 296         WSBMLISTADD(&kbo->head, &cache->cached);
 297     else
 298         WSBMLISTADD(&kbo->head, &cache->unCached);
 299
 300     WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList);
 301     wsbmFreeTimeoutKBOsLocked(cache, &time);
 302
 303     WSBM_MUTEX_UNLOCK(&cache->mutex);
 304 }
 305
 306 /*
 307  * Get a _SlabKernelBO for us to use as storage for a slab.
 308  */
 309
 310 static struct _WsbmSlabKernelBO *
 311 wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
 312 {
 313     struct _WsbmSlabPool *slabPool = header->slabPool;
 314     struct _WsbmSlabCache *cache = slabPool->cache;
 315     struct _WsbmListHead *list, *head;
 316     uint32_t size = header->bufSize * slabPool->desiredNumBuffers;
 317     struct _WsbmSlabKernelBO *kbo;
 318     struct _WsbmSlabKernelBO *kboTmp;
 319     int ret;
 320
 321     /*
 322      * FIXME: We should perhaps allow some variation in slabsize in order
 323      * to efficiently reuse slabs.
 324      */
 325
 326     size = (size <= slabPool->maxSlabSize) ? size : slabPool->maxSlabSize;
 327     if (size < header->bufSize)
 328         size = header->bufSize;
 329     size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1);
 330     WSBM_MUTEX_LOCK(&cache->mutex);
 331
 332     kbo = NULL;
 333
 334   retry:
 335     head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ?
 336         &cache->cached : &cache->unCached;
 337
 338     WSBMLISTFOREACH(list, head) {
 339         kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head);
 340
 341         if ((kboTmp->actualSize == size) &&
 342             (slabPool->pageAlignment == 0 ||
 343              (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) {
 344
 345             if (!kbo)
 346                 kbo = kboTmp;
 347
 348             if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0)
 349                 break;
 350
 351         }
 352     }
 353
 354     if (kbo) {
 355         WSBMLISTDELINIT(&kbo->head);
 356         WSBMLISTDELINIT(&kbo->timeoutHead);
 357     }
 358
 359     WSBM_MUTEX_UNLOCK(&cache->mutex);
 360
 361     if (kbo) {
 362         uint32_t new_mask =
 363             kbo->proposedPlacement ^ slabPool->proposedPlacement;
 364
 365         ret = 0;
 366         if (new_mask) {
 367             union ttm_pl_setstatus_arg arg;
 368             struct ttm_pl_setstatus_req *req = &arg.req;
 369             struct ttm_pl_rep *rep = &arg.rep;
 370
 371             req->handle = kbo->kBuf.handle;
 372             req->set_placement = slabPool->proposedPlacement & new_mask;
 373             req->clr_placement = ~slabPool->proposedPlacement & new_mask;
 374             DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
 375                                        slabPool->devOffset + TTM_PL_SETSTATUS,
 376                                        arg, ret);
 377             if (ret == 0) {
 378                 kbo->kBuf.gpuOffset = rep->gpu_offset;
 379                 kbo->kBuf.placement = rep->placement;
 380             }
 381             kbo->proposedPlacement = slabPool->proposedPlacement;
 382         }
 383
 384         if (ret == 0)
 385             return kbo;
 386
 387         wsbmFreeKernelBO(kbo);
 388         kbo = NULL;
 389         goto retry;
 390     }
 391
 392     kbo = calloc(1, sizeof(*kbo));
 393     if (!kbo)
 394         return NULL;
 395
 396     {
 397         union ttm_pl_create_arg arg;
 398
 399         kbo->slabPool = slabPool;
 400         WSBMINITLISTHEAD(&kbo->head);
 401         WSBMINITLISTHEAD(&kbo->timeoutHead);
 402
 403         arg.req.size = size;
 404         arg.req.placement = slabPool->proposedPlacement;
 405         arg.req.page_alignment = slabPool->pageAlignment;
 406
 407         DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
 408                                    slabPool->devOffset + TTM_PL_CREATE,
 409                                    arg, ret);
 410         if (ret)
 411             goto out_err0;
 412
 413         kbo->kBuf.gpuOffset = arg.rep.gpu_offset;
 414         kbo->kBuf.placement = arg.rep.placement;
 415         kbo->kBuf.handle = arg.rep.handle;
 416
 417         kbo->actualSize = arg.rep.bo_size;
 418         kbo->mapHandle = arg.rep.map_handle;
 419         kbo->proposedPlacement = slabPool->proposedPlacement;
 420     }
 421
 422     kbo->virtual = mmap(0, kbo->actualSize,
 423                         PROT_READ | PROT_WRITE, MAP_SHARED,
 424                         slabPool->pool.fd, kbo->mapHandle);
 425
 426     if (kbo->virtual == MAP_FAILED) {
 427         ret = -errno;
 428         goto out_err1;
 429     }
 430
 431     return kbo;
 432
 433   out_err1:
 434     {
 435         struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle };
 436
 437         (void)drmCommandWrite(slabPool->pool.fd,
 438                               slabPool->devOffset + TTM_PL_UNREF,
 439                               &arg, sizeof(arg));
 440     }
 441   out_err0:
 442     free(kbo);
 443     return NULL;
 444 }
 445
 446 static int
 447 wsbmAllocSlab(struct _WsbmSlabSizeHeader *header)
 448 {
 449     struct _WsbmSlab *slab;
 450     struct _WsbmSlabBuffer *sBuf;
 451     uint32_t numBuffers;
 452     int ret;
 453     int i;
 454
 455     slab = calloc(1, sizeof(*slab));
 456     if (!slab)
 457         return -ENOMEM;
 458
 459     slab->kbo = wsbmAllocKernelBO(header);
 460     if (!slab->kbo) {
 461         ret = -ENOMEM;
 462         goto out_err0;
 463     }
 464
 465     numBuffers = slab->kbo->actualSize / header->bufSize;
 466
 467     slab->buffers = calloc(numBuffers, sizeof(*slab->buffers));
 468     if (!slab->buffers) {
 469         ret = -ENOMEM;
 470         goto out_err1;
 471     }
 472
 473     WSBMINITLISTHEAD(&slab->head);
 474     WSBMINITLISTHEAD(&slab->freeBuffers);
 475     slab->numBuffers = numBuffers;
 476     slab->numFree = 0;
 477     slab->header = header;
 478
 479     sBuf = slab->buffers;
 480     for (i = 0; i < numBuffers; ++i) {
 481         ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool);
 482         if (ret)
 483             goto out_err2;
 484         sBuf->parent = slab;
 485         sBuf->start = i * header->bufSize;
 486         sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual +
 487                                  sBuf->start);
 488         wsbmAtomicSet(&sBuf->writers, 0);
 489         sBuf->isSlabBuffer = 1;
 490         WSBM_COND_INIT(&sBuf->event);
 491         WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers);
 492         slab->numFree++;
 493         sBuf++;
 494     }
 495
 496     WSBMLISTADDTAIL(&slab->head, &header->slabs);
 497
 498     return 0;
 499
 500   out_err2:
 501     sBuf = slab->buffers;
 502     for (i = 0; i < numBuffers; ++i) {
 503         if (sBuf->parent == slab) {
 504             WSBM_COND_FREE(&sBuf->event);
 505             wsbmBufStorageTakedown(&sBuf->storage);
 506         }
 507         sBuf++;
 508     }
 509     free(slab->buffers);
 510   out_err1:
 511     wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo);
 512   out_err0:
 513     free(slab);
 514     return ret;
 515 }
 516
 517 /*
 518  * Delete a buffer from the slab header delayed list and put
 519  * it on the slab free list.
 520  */
 521
 522 static void
 523 wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf)
 524 {
 525     struct _WsbmSlab *slab = buf->parent;
 526     struct _WsbmSlabSizeHeader *header = slab->header;
 527     struct _WsbmListHead *list = &buf->head;
 528
 529     WSBMLISTDEL(list);
 530     WSBMLISTADDTAIL(list, &slab->freeBuffers);
 531     slab->numFree++;
 532
 533     if (slab->head.next == &slab->head)
 534         WSBMLISTADDTAIL(&slab->head, &header->slabs);
 535
 536     if (slab->numFree == slab->numBuffers) {
 537         list = &slab->head;
 538         WSBMLISTDEL(list);
 539         WSBMLISTADDTAIL(list, &header->freeSlabs);
 540     }
 541
 542     if (header->slabs.next == &header->slabs ||
 543         slab->numFree != slab->numBuffers) {
 544
 545         struct _WsbmListHead *next;
 546         struct _WsbmSlabCache *cache = header->slabPool->cache;
 547
 548         WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) {
 549             int i;
 550             struct _WsbmSlabBuffer *sBuf;
 551
 552             slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
 553
 554             WSBMLISTDELINIT(list);
 555
 556             sBuf = slab->buffers;
 557             for (i = 0; i < slab->numBuffers; ++i) {
 558                 if (sBuf->parent == slab) {
 559                     WSBM_COND_FREE(&sBuf->event);
 560                     wsbmBufStorageTakedown(&sBuf->storage);
 561                 }
 562                 sBuf++;
 563             }
 564             wsbmSetKernelBOFree(cache, slab->kbo);
 565             free(slab->buffers);
 566             free(slab);
 567         }
 568     }
 569 }
 570
 571 static void
 572 wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait)
 573 {
 574   struct _WsbmListHead *list, *prev, *first, *head;
 575     struct _WsbmSlabBuffer *sBuf;
 576     struct _WsbmSlab *slab;
 577     int firstWasSignaled = 1;
 578     int signaled;
 579     int i;
 580     int ret;
 581
 582     /*
 583      * Rerun the freeing test if the youngest tested buffer
 584      * was signaled, since there might be more idle buffers
 585      * in the delay list.
 586      */
 587
 588     while (firstWasSignaled) {
 589         firstWasSignaled = 0;
 590         signaled = 0;
 591         first = header->delayedBuffers.next;
 592
 593         /* Only examine the oldest 1/3 of delayed buffers:
 594          */
 595         if (header->numDelayed > 3) {
 596             for (i = 0; i < header->numDelayed; i += 3) {
 597                 first = first->next;
 598             }
 599         }
 600
 601         /*
 602          * No need to take the buffer mutex for each buffer we loop
 603          * through since we're currently the only user.
 604          */
 605
 606         head = first->next;
 607         WSBMLISTFOREACHPREVSAFE(list, prev, head) {
 608
 609             if (list == &header->delayedBuffers)
 610                 break;
 611
 612             sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
 613
 614             slab = sBuf->parent;
 615
 616             if (!signaled) {
 617                 if (wait) {
 618                     ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
 619                     if (ret)
 620                         break;
 621                     signaled = 1;
 622                     wait = 0;
 623                 } else {
 624                     signaled =
 625                         wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
 626 #ifdef DEBUG_FENCESIGNALED
 627                     fencesignaled++;
 628 #endif
 629                 }
 630                 if (signaled) {
 631                     if (list == first)
 632                         firstWasSignaled = 1;
 633                     wsbmFenceUnreference(&sBuf->fence);
 634                     header->numDelayed--;
 635                     wsbmSlabFreeBufferLocked(sBuf);
 636                 } else
 637                     break;
 638             } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
 639                 wsbmFenceUnreference(&sBuf->fence);
 640                 header->numDelayed--;
 641                 wsbmSlabFreeBufferLocked(sBuf);
 642             }
 643         }
 644     }
 645 }
 646
 647 static struct _WsbmSlabBuffer *
 648 wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header)
 649 {
 650     static struct _WsbmSlabBuffer *buf;
 651     struct _WsbmSlab *slab;
 652     struct _WsbmListHead *list;
 653     int count = WSBM_SLABPOOL_ALLOC_RETRIES;
 654
 655     WSBM_MUTEX_LOCK(&header->mutex);
 656     while (header->slabs.next == &header->slabs && count > 0) {
 657         wsbmSlabCheckFreeLocked(header, 0);
 658         if (header->slabs.next != &header->slabs)
 659             break;
 660
 661         WSBM_MUTEX_UNLOCK(&header->mutex);
 662         if (count != WSBM_SLABPOOL_ALLOC_RETRIES)
 663             usleep(1000);
 664         WSBM_MUTEX_LOCK(&header->mutex);
 665         (void)wsbmAllocSlab(header);
 666         count--;
 667     }
 668
 669     list = header->slabs.next;
 670     if (list == &header->slabs) {
 671         WSBM_MUTEX_UNLOCK(&header->mutex);
 672         return NULL;
 673     }
 674     slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
 675     if (--slab->numFree == 0)
 676         WSBMLISTDELINIT(list);
 677
 678     list = slab->freeBuffers.next;
 679     WSBMLISTDELINIT(list);
 680
 681     WSBM_MUTEX_UNLOCK(&header->mutex);
 682     buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
 683
 684     buf->storage.destroyContainer = NULL;
 685
 686 #ifdef DEBUG_FENCESIGNALED
 687     createbuffer++;
 688 #endif
 689     return buf;
 690 }
 691
 692 static struct _WsbmBufStorage *
 693 pool_create(struct _WsbmBufferPool *pool, unsigned long size,
 694             uint32_t placement, unsigned alignment)
 695 {
 696     struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
 697     struct _WsbmSlabSizeHeader *header;
 698     struct _WsbmSlabBuffer *sBuf;
 699     int i;
 700     int ret;
 701
 702     /*
 703      * FIXME: Check for compatibility.
 704      */
 705
 706     header = slabPool->headers;
 707     for (i = 0; i < slabPool->numBuckets; ++i) {
 708         if (header->bufSize >= size)
 709             break;
 710         header++;
 711     }
 712
 713     if (i < slabPool->numBuckets) {
 714         sBuf = wsbmSlabAllocBuffer(header);
 715         return ((sBuf) ? &sBuf->storage : NULL);
 716     }
 717
 718     /*
 719      * Fall back to allocate a buffer object directly from DRM.
 720      * and wrap it in a wsbmBO structure.
 721      */
 722
 723     sBuf = calloc(1, sizeof(*sBuf));
 724
 725     if (!sBuf)
 726         return NULL;
 727
 728     if (alignment) {
 729         if ((alignment < slabPool->pageSize)
 730             && (slabPool->pageSize % alignment))
 731             goto out_err0;
 732         if ((alignment > slabPool->pageSize)
 733             && (alignment % slabPool->pageSize))
 734             goto out_err0;
 735     }
 736
 737     ret = wsbmBufStorageInit(&sBuf->storage, pool);
 738     if (ret)
 739         goto out_err0;
 740
 741     ret = WSBM_COND_INIT(&sBuf->event);
 742     if (ret)
 743         goto out_err1;
 744
 745     {
 746         union ttm_pl_create_arg arg;
 747
 748         arg.req.size = size;
 749         arg.req.placement = placement;
 750         arg.req.page_alignment = alignment / slabPool->pageSize;
 751
 752         DRMRESTARTCOMMANDWRITEREAD(pool->fd,
 753                                    slabPool->devOffset + TTM_PL_CREATE,
 754                                    arg, ret);
 755
 756         if (ret)
 757             goto out_err2;
 758
 759         sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
 760         sBuf->kBuf.placement = arg.rep.placement;
 761         sBuf->kBuf.handle = arg.rep.handle;
 762         sBuf->mapHandle = arg.rep.map_handle;
 763         sBuf->requestedSize = size;
 764
 765         sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
 766                              pool->fd, sBuf->mapHandle);
 767
 768         if (sBuf->virtual == MAP_FAILED)
 769             goto out_err3;
 770     }
 771
 772     wsbmAtomicSet(&sBuf->writers, 0);
 773     return &sBuf->storage;
 774   out_err3:
 775     {
 776         struct ttm_pl_reference_req arg;
 777
 778         arg.handle = sBuf->kBuf.handle;
 779         (void)drmCommandWriteRead(pool->fd,
 780                                   slabPool->devOffset + TTM_PL_UNREF,
 781                                   &arg, sizeof(arg));
 782     }
 783   out_err2:
 784     WSBM_COND_FREE(&sBuf->event);
 785   out_err1:
 786     wsbmBufStorageTakedown(&sBuf->storage);
 787   out_err0:
 788     free(sBuf);
 789     return NULL;
 790 }
 791
 792 static void
 793 pool_destroy(struct _WsbmBufStorage **p_buf)
 794 {
 795     struct _WsbmBufStorage *buf = *p_buf;
 796     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 797     struct _WsbmSlab *slab;
 798     struct _WsbmSlabSizeHeader *header;
 799
 800     *p_buf = NULL;
 801
 802     if (!sBuf->isSlabBuffer) {
 803         struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);
 804         struct ttm_pl_reference_req arg;
 805
 806         if (sBuf->virtual != NULL) {
 807             (void)munmap(sBuf->virtual, sBuf->requestedSize);
 808             sBuf->virtual = NULL;
 809         }
 810
 811         arg.handle = sBuf->kBuf.handle;
 812         (void)drmCommandWrite(slabPool->pool.fd,
 813                               slabPool->devOffset + TTM_PL_UNREF,
 814                               &arg, sizeof(arg));
 815
 816         WSBM_COND_FREE(&sBuf->event);
 817         wsbmBufStorageTakedown(&sBuf->storage);
 818         free(sBuf);
 819         return;
 820     }
 821
 822     slab = sBuf->parent;
 823     header = slab->header;
 824
 825     /*
 826      * No need to take the buffer mutex below since we're the only user.
 827      */
 828
 829     WSBM_MUTEX_LOCK(&header->mutex);
 830     sBuf->unFenced = 0;
 831     wsbmAtomicSet(&sBuf->writers, 0);
 832     wsbmAtomicSet(&sBuf->storage.refCount, 1);
 833
 834     if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
 835         WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers);
 836         header->numDelayed++;
 837     } else {
 838         if (sBuf->fence)
 839             wsbmFenceUnreference(&sBuf->fence);
 840         wsbmSlabFreeBufferLocked(sBuf);
 841     }
 842
 843     WSBM_MUTEX_UNLOCK(&header->mutex);
 844 }
 845
 846 static void
 847 waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy)
 848 {
 849     struct _WsbmBufStorage *storage = &sBuf->storage;
 850
 851     while (sBuf->unFenced || sBuf->fence != NULL) {
 852
 853         if (sBuf->unFenced)
 854             WSBM_COND_WAIT(&sBuf->event, &storage->mutex);
 855
 856         if (sBuf->fence != NULL) {
 857             if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) {
 858                 struct _WsbmFenceObject *fence =
 859                     wsbmFenceReference(sBuf->fence);
 860
 861                 WSBM_MUTEX_UNLOCK(&storage->mutex);
 862                 (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy);
 863                 WSBM_MUTEX_LOCK(&storage->mutex);
 864                 if (sBuf->fence == fence)
 865                     wsbmFenceUnreference(&sBuf->fence);
 866
 867                 wsbmFenceUnreference(&fence);
 868             } else {
 869                 wsbmFenceUnreference(&sBuf->fence);
 870             }
 871         }
 872     }
 873 }
 874
 875 static int
 876 pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
 877 {
 878     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 879
 880     WSBM_MUTEX_LOCK(&buf->mutex);
 881     waitIdleLocked(sBuf, lazy);
 882     WSBM_MUTEX_UNLOCK(&buf->mutex);
 883
 884     return 0;
 885 }
 886
 887 static int
 888 pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual)
 889 {
 890     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 891
 892     *virtual = sBuf->virtual;
 893
 894     return 0;
 895 }
 896
 897 static void
 898 pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode)
 899 {
 900     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 901
 902     if (wsbmAtomicDecZero(&sBuf->writers))
 903         WSBM_COND_BROADCAST(&sBuf->event);
 904 }
 905
 906 static int
 907 pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
 908 {
 909     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 910     int ret = 0;
 911
 912     WSBM_MUTEX_LOCK(&buf->mutex);
 913     if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
 914         int signaled;
 915
 916         if (sBuf->unFenced) {
 917             ret = -EBUSY;
 918             goto out_unlock;
 919         }
 920
 921         if (sBuf->isSlabBuffer)
 922             signaled = (sBuf->fence == NULL) ||
 923                 wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType);
 924         else
 925             signaled = (sBuf->fence == NULL) ||
 926                 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
 927
 928         ret = 0;
 929         if (signaled) {
 930             wsbmFenceUnreference(&sBuf->fence);
 931             wsbmAtomicInc(&sBuf->writers);
 932         } else
 933             ret = -EBUSY;
 934         goto out_unlock;
 935     }
 936     waitIdleLocked(sBuf, 0);
 937     wsbmAtomicInc(&sBuf->writers);
 938   out_unlock:
 939     WSBM_MUTEX_UNLOCK(&buf->mutex);
 940     return ret;
 941 }
 942
 943 static void
 944 pool_unmap(struct _WsbmBufStorage *buf)
 945 {
 946     ;
 947 }
 948
 949 static unsigned long
 950 pool_poolOffset(struct _WsbmBufStorage *buf)
 951 {
 952     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 953
 954     return sBuf->start;
 955 }
 956
 957 static unsigned long
 958 pool_size(struct _WsbmBufStorage *buf)
 959 {
 960     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 961
 962     if (!sBuf->isSlabBuffer)
 963         return sBuf->requestedSize;
 964
 965     return sBuf->parent->header->bufSize;
 966 }
 967
 968 static struct _WsbmKernelBuf *
 969 pool_kernel(struct _WsbmBufStorage *buf)
 970 {
 971     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 972
 973     return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;
 974 }
 975
 976 static unsigned long
 977 pool_offset(struct _WsbmBufStorage *buf)
 978 {
 979     return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf);
 980 }
 981
 982 static void
 983 pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
 984 {
 985     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
 986     struct _WsbmKernelBuf *kBuf;
 987
 988     WSBM_MUTEX_LOCK(&buf->mutex);
 989     if (sBuf->fence)
 990         wsbmFenceUnreference(&sBuf->fence);
 991
 992     kBuf = pool_kernel(buf);
 993     sBuf->fenceType = kBuf->fence_type_mask;
 994     if (!wsbmFenceSignaledCached(fence, sBuf->fenceType))
 995         sBuf->fence = wsbmFenceReference(fence);
 996
 997     sBuf->unFenced = 0;
 998     WSBM_COND_BROADCAST(&sBuf->event);
 999     WSBM_MUTEX_UNLOCK(&buf->mutex);
1000 }
1001
1002 static int
1003 pool_validate(struct _WsbmBufStorage *buf,
1004               uint64_t set_flags, uint64_t clr_flags)
1005 {
1006     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1007
1008     WSBM_MUTEX_LOCK(&buf->mutex);
1009     while (wsbmAtomicRead(&sBuf->writers) != 0) {
1010         WSBM_COND_WAIT(&sBuf->event, &buf->mutex);
1011     }
1012
1013     sBuf->unFenced = 1;
1014     WSBM_MUTEX_UNLOCK(&buf->mutex);
1015     return 0;
1016 }
1017
1018 static void
1019 pool_unvalidate(struct _WsbmBufStorage *buf)
1020 {
1021     struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1022
1023     WSBM_MUTEX_LOCK(&buf->mutex);
1024     if (sBuf->unFenced) {
1025         sBuf->unFenced = 0;
1026         WSBM_COND_BROADCAST(&sBuf->event);
1027     }
1028     WSBM_MUTEX_UNLOCK(&buf->mutex);
1029 }
1030
1031 struct _WsbmSlabCache *
1032 wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec)
1033 {
1034     struct _WsbmSlabCache *tmp;
1035
1036     tmp = calloc(1, sizeof(*tmp));
1037     if (!tmp)
1038         return NULL;
1039
1040     WSBM_MUTEX_INIT(&tmp->mutex);
1041     WSBM_MUTEX_LOCK(&tmp->mutex);
1042     tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000;
1043     tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000;
1044     tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000;
1045
1046     tmp->checkInterval.tv_usec = checkIntervalMsec * 1000;
1047     tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000;
1048     tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000;
1049
1050     gettimeofday(&tmp->nextCheck, NULL);
1051     wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval);
1052     WSBMINITLISTHEAD(&tmp->timeoutList);
1053     WSBMINITLISTHEAD(&tmp->unCached);
1054     WSBMINITLISTHEAD(&tmp->cached);
1055     WSBM_MUTEX_UNLOCK(&tmp->mutex);
1056
1057     return tmp;
1058 }
1059
1060 void
1061 wsbmSlabCacheFinish(struct _WsbmSlabCache *cache)
1062 {
1063     struct timeval time;
1064
1065     time = cache->nextCheck;
1066     WSBM_MUTEX_LOCK(&cache->mutex);
1067     wsbmTimeAdd(&time, &cache->checkInterval);
1068     wsbmFreeTimeoutKBOsLocked(cache, &time);
1069     WSBM_MUTEX_UNLOCK(&cache->mutex);
1070
1071     assert(cache->timeoutList.next == &cache->timeoutList);
1072     assert(cache->unCached.next == &cache->unCached);
1073     assert(cache->cached.next == &cache->cached);
1074
1075     WSBM_MUTEX_FREE(&cache->mutex);
1076     free(cache);
1077 }
1078
1079 static void
1080 wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size,
1081                    struct _WsbmSlabSizeHeader *header)
1082 {
1083     WSBM_MUTEX_INIT(&header->mutex);
1084     WSBM_MUTEX_LOCK(&header->mutex);
1085
1086     WSBMINITLISTHEAD(&header->slabs);
1087     WSBMINITLISTHEAD(&header->freeSlabs);
1088     WSBMINITLISTHEAD(&header->delayedBuffers);
1089
1090     header->numDelayed = 0;
1091     header->slabPool = slabPool;
1092     header->bufSize = size;
1093
1094     WSBM_MUTEX_UNLOCK(&header->mutex);
1095 }
1096
1097 static void
1098 wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header)
1099 {
1100     struct _WsbmListHead *list, *next;
1101     struct _WsbmSlabBuffer *sBuf;
1102
1103     WSBM_MUTEX_LOCK(&header->mutex);
1104     WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) {
1105         sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
1106
1107         if (sBuf->fence) {
1108             (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
1109             wsbmFenceUnreference(&sBuf->fence);
1110         }
1111         header->numDelayed--;
1112         wsbmSlabFreeBufferLocked(sBuf);
1113     }
1114     WSBM_MUTEX_UNLOCK(&header->mutex);
1115     WSBM_MUTEX_FREE(&header->mutex);
1116 }
1117
1118 static void
1119 pool_takedown(struct _WsbmBufferPool *pool)
1120 {
1121     struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
1122     int i;
1123
1124     for (i = 0; i < slabPool->numBuckets; ++i) {
1125         wsbmFinishSizeHeader(&slabPool->headers[i]);
1126     }
1127
1128     free(slabPool->headers);
1129     free(slabPool->bucketSizes);
1130     free(slabPool);
1131 }
1132
1133 struct _WsbmBufferPool *
1134 wsbmSlabPoolInit(int fd,
1135                  uint32_t devOffset,
1136                  uint32_t placement,
1137                  uint32_t validMask,
1138                  uint32_t smallestSize,
1139                  uint32_t numSizes,
1140                  uint32_t desiredNumBuffers,
1141                  uint32_t maxSlabSize,
1142                  uint32_t pageAlignment, struct _WsbmSlabCache *cache)
1143 {
1144     struct _WsbmBufferPool *pool;
1145     struct _WsbmSlabPool *slabPool;
1146     uint32_t i;
1147
1148     slabPool = calloc(1, sizeof(*slabPool));
1149     if (!slabPool)
1150         return NULL;
1151
1152     pool = &slabPool->pool;
1153
1154     slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes));
1155     if (!slabPool->bucketSizes)
1156         goto out_err0;
1157
1158     slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers));
1159     if (!slabPool->headers)
1160         goto out_err1;
1161
1162     slabPool->devOffset = devOffset;
1163     slabPool->cache = cache;
1164     slabPool->proposedPlacement = placement;
1165     slabPool->validMask = validMask;
1166     slabPool->numBuckets = numSizes;
1167     slabPool->pageSize = getpagesize();
1168     slabPool->pageAlignment = pageAlignment;
1169     slabPool->maxSlabSize = maxSlabSize;
1170     slabPool->desiredNumBuffers = desiredNumBuffers;
1171
1172     for (i = 0; i < slabPool->numBuckets; ++i) {
1173         slabPool->bucketSizes[i] = (smallestSize << i);
1174         wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i],
1175                            &slabPool->headers[i]);
1176     }
1177
1178     pool->fd = fd;
1179     pool->map = &pool_map;
1180     pool->unmap = &pool_unmap;
1181     pool->destroy = &pool_destroy;
1182     pool->offset = &pool_offset;
1183     pool->poolOffset = &pool_poolOffset;
1184     pool->size = &pool_size;
1185     pool->create = &pool_create;
1186     pool->fence = &pool_fence;
1187     pool->kernel = &pool_kernel;
1188     pool->validate = &pool_validate;
1189     pool->unvalidate = &pool_unvalidate;
1190     pool->waitIdle = &pool_waitIdle;
1191     pool->takeDown = &pool_takedown;
1192     pool->releasefromcpu = &pool_releaseFromCpu;
1193     pool->syncforcpu = &pool_syncForCpu;
1194
1195     return pool;
1196
1197   out_err1:
1198     free(slabPool->bucketSizes);
1199   out_err0:
1200     free(slabPool);
1201
1202     return NULL;
1203 }