drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_bo.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <drm/ttm/ttm_tt.h>
  37
  38 #include <linux/jiffies.h>
  39 #include <linux/slab.h>
  40 #include <linux/sched.h>
  41 #include <linux/mm.h>
  42 #include <linux/file.h>
  43 #include <linux/module.h>
  44 #include <linux/atomic.h>
  45 #include <linux/dma-resv.h>
  46
  47 #include "ttm_module.h"
  48
  49 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  50                                         struct ttm_placement *placement)
  51 {
  52         struct drm_printer p = drm_debug_printer(TTM_PFX);
  53         struct ttm_resource_manager *man;
  54         int i, mem_type;
  55
  56         for (i = 0; i < placement->num_placement; i++) {
  57                 mem_type = placement->placement[i].mem_type;
  58                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  59                            i, placement->placement[i].flags, mem_type);
  60                 man = ttm_manager_type(bo->bdev, mem_type);
  61                 ttm_resource_manager_debug(man, &p);
  62         }
  63 }
  64
  65 /**
  66  * ttm_bo_move_to_lru_tail
  67  *
  68  * @bo: The buffer object.
  69  *
  70  * Move this BO to the tail of all lru lists used to lookup and reserve an
  71  * object. This function must be called with struct ttm_global::lru_lock
  72  * held, and is used to make a BO less likely to be considered for eviction.
  73  */
  74 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
  75 {
  76         dma_resv_assert_held(bo->base.resv);
  77
  78         if (bo->resource)
  79                 ttm_resource_move_to_lru_tail(bo->resource);
  80 }
  81 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
  82
  83 /**
  84  * ttm_bo_set_bulk_move - update BOs bulk move object
  85  *
  86  * @bo: The buffer object.
  87  *
  88  * Update the BOs bulk move object, making sure that resources are added/removed
  89  * as well. A bulk move allows to move many resource on the LRU at once,
  90  * resulting in much less overhead of maintaining the LRU.
  91  * The only requirement is that the resources stay together on the LRU and are
  92  * never separated. This is enforces by setting the bulk_move structure on a BO.
  93  * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
  94  * their LRU list.
  95  */
  96 void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
  97                           struct ttm_lru_bulk_move *bulk)
  98 {
  99         dma_resv_assert_held(bo->base.resv);
 100
 101         if (bo->bulk_move == bulk)
 102                 return;
 103
 104         spin_lock(&bo->bdev->lru_lock);
 105         if (bo->resource)
 106                 ttm_resource_del_bulk_move(bo->resource, bo);
 107         bo->bulk_move = bulk;
 108         if (bo->resource)
 109                 ttm_resource_add_bulk_move(bo->resource, bo);
 110         spin_unlock(&bo->bdev->lru_lock);
 111 }
 112 EXPORT_SYMBOL(ttm_bo_set_bulk_move);
 113
 114 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 115                                   struct ttm_resource *mem, bool evict,
 116                                   struct ttm_operation_ctx *ctx,
 117                                   struct ttm_place *hop)
 118 {
 119         struct ttm_device *bdev = bo->bdev;
 120         bool old_use_tt, new_use_tt;
 121         int ret;
 122
 123         old_use_tt = bo->resource &&
 124                 ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
 125         new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
 126
 127         ttm_bo_unmap_virtual(bo);
 128
 129         /*
 130          * Create and bind a ttm if required.
 131          */
 132
 133         if (new_use_tt) {
 134                 /* Zero init the new TTM structure if the old location should
 135                  * have used one as well.
 136                  */
 137                 ret = ttm_tt_create(bo, old_use_tt);
 138                 if (ret)
 139                         goto out_err;
 140
 141                 if (mem->mem_type != TTM_PL_SYSTEM) {
 142                         ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
 143                         if (ret)
 144                                 goto out_err;
 145                 }
 146         }
 147
 148         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 149         if (ret)
 150                 goto out_err;
 151
 152         ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
 153         if (ret) {
 154                 if (ret == -EMULTIHOP)
 155                         return ret;
 156                 goto out_err;
 157         }
 158
 159         ctx->bytes_moved += bo->base.size;
 160         return 0;
 161
 162 out_err:
 163         if (!old_use_tt)
 164                 ttm_bo_tt_destroy(bo);
 165
 166         return ret;
 167 }
 168
 169 /*
 170  * Call bo::reserved.
 171  * Will release GPU memory type usage on destruction.
 172  * This is the place to put in driver specific hooks to release
 173  * driver private resources.
 174  * Will release the bo::reserved lock.
 175  */
 176
 177 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 178 {
 179         if (bo->bdev->funcs->delete_mem_notify)
 180                 bo->bdev->funcs->delete_mem_notify(bo);
 181
 182         ttm_bo_tt_destroy(bo);
 183         ttm_resource_free(bo, &bo->resource);
 184 }
 185
 186 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 187 {
 188         int r;
 189
 190         if (bo->base.resv == &bo->base._resv)
 191                 return 0;
 192
 193         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 194
 195         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 196         dma_resv_unlock(&bo->base._resv);
 197         if (r)
 198                 return r;
 199
 200         if (bo->type != ttm_bo_type_sg) {
 201                 /* This works because the BO is about to be destroyed and nobody
 202                  * reference it any more. The only tricky case is the trylock on
 203                  * the resv object while holding the lru_lock.
 204                  */
 205                 spin_lock(&bo->bdev->lru_lock);
 206                 bo->base.resv = &bo->base._resv;
 207                 spin_unlock(&bo->bdev->lru_lock);
 208         }
 209
 210         return r;
 211 }
 212
 213 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 214 {
 215         struct dma_resv *resv = &bo->base._resv;
 216         struct dma_resv_iter cursor;
 217         struct dma_fence *fence;
 218
 219         dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
 220         dma_resv_for_each_fence_unlocked(&cursor, fence) {
 221                 if (!fence->ops->signaled)
 222                         dma_fence_enable_sw_signaling(fence);
 223         }
 224         dma_resv_iter_end(&cursor);
 225 }
 226
 227 /**
 228  * ttm_bo_cleanup_refs
 229  * If bo idle, remove from lru lists, and unref.
 230  * If not idle, block if possible.
 231  *
 232  * Must be called with lru_lock and reservation held, this function
 233  * will drop the lru lock and optionally the reservation lock before returning.
 234  *
 235  * @bo:                    The buffer object to clean-up
 236  * @interruptible:         Any sleeps should occur interruptibly.
 237  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
 238  * @unlock_resv:           Unlock the reservation lock as well.
 239  */
 240
 241 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 242                                bool interruptible, bool no_wait_gpu,
 243                                bool unlock_resv)
 244 {
 245         struct dma_resv *resv = &bo->base._resv;
 246         int ret;
 247
 248         if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
 249                 ret = 0;
 250         else
 251                 ret = -EBUSY;
 252
 253         if (ret && !no_wait_gpu) {
 254                 long lret;
 255
 256                 if (unlock_resv)
 257                         dma_resv_unlock(bo->base.resv);
 258                 spin_unlock(&bo->bdev->lru_lock);
 259
 260                 lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
 261                                              interruptible,
 262                                              30 * HZ);
 263
 264                 if (lret < 0)
 265                         return lret;
 266                 else if (lret == 0)
 267                         return -EBUSY;
 268
 269                 spin_lock(&bo->bdev->lru_lock);
 270                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 271                         /*
 272                          * We raced, and lost, someone else holds the reservation now,
 273                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 274                          *
 275                          * Even if it's not the case, because we finished waiting any
 276                          * delayed destruction would succeed, so just return success
 277                          * here.
 278                          */
 279                         spin_unlock(&bo->bdev->lru_lock);
 280                         return 0;
 281                 }
 282                 ret = 0;
 283         }
 284
 285         if (ret) {
 286                 if (unlock_resv)
 287                         dma_resv_unlock(bo->base.resv);
 288                 spin_unlock(&bo->bdev->lru_lock);
 289                 return ret;
 290         }
 291
 292         spin_unlock(&bo->bdev->lru_lock);
 293         ttm_bo_cleanup_memtype_use(bo);
 294
 295         if (unlock_resv)
 296                 dma_resv_unlock(bo->base.resv);
 297
 298         return 0;
 299 }
 300
 301 /*
 302  * Block for the dma_resv object to become idle, lock the buffer and clean up
 303  * the resource and tt object.
 304  */
 305 static void ttm_bo_delayed_delete(struct work_struct *work)
 306 {
 307         struct ttm_buffer_object *bo;
 308
 309         bo = container_of(work, typeof(*bo), delayed_delete);
 310
 311         dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
 312                               MAX_SCHEDULE_TIMEOUT);
 313         dma_resv_lock(bo->base.resv, NULL);
 314         ttm_bo_cleanup_memtype_use(bo);
 315         dma_resv_unlock(bo->base.resv);
 316         ttm_bo_put(bo);
 317 }
 318
 319 static void ttm_bo_release(struct kref *kref)
 320 {
 321         struct ttm_buffer_object *bo =
 322             container_of(kref, struct ttm_buffer_object, kref);
 323         struct ttm_device *bdev = bo->bdev;
 324         int ret;
 325
 326         WARN_ON_ONCE(bo->pin_count);
 327         WARN_ON_ONCE(bo->bulk_move);
 328
 329         if (!bo->deleted) {
 330                 ret = ttm_bo_individualize_resv(bo);
 331                 if (ret) {
 332                         /* Last resort, if we fail to allocate memory for the
 333                          * fences block for the BO to become idle
 334                          */
 335                         dma_resv_wait_timeout(bo->base.resv,
 336                                               DMA_RESV_USAGE_BOOKKEEP, false,
 337                                               30 * HZ);
 338                 }
 339
 340                 if (bo->bdev->funcs->release_notify)
 341                         bo->bdev->funcs->release_notify(bo);
 342
 343                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 344                 ttm_mem_io_free(bdev, bo->resource);
 345
 346                 if (!dma_resv_test_signaled(bo->base.resv,
 347                                             DMA_RESV_USAGE_BOOKKEEP) ||
 348                     !dma_resv_trylock(bo->base.resv)) {
 349                         /* The BO is not idle, resurrect it for delayed destroy */
 350                         ttm_bo_flush_all_fences(bo);
 351                         bo->deleted = true;
 352
 353                         spin_lock(&bo->bdev->lru_lock);
 354
 355                         /*
 356                          * Make pinned bos immediately available to
 357                          * shrinkers, now that they are queued for
 358                          * destruction.
 359                          *
 360                          * FIXME: QXL is triggering this. Can be removed when the
 361                          * driver is fixed.
 362                          */
 363                         if (bo->pin_count) {
 364                                 bo->pin_count = 0;
 365                                 ttm_resource_move_to_lru_tail(bo->resource);
 366                         }
 367
 368                         kref_init(&bo->kref);
 369                         spin_unlock(&bo->bdev->lru_lock);
 370
 371                         INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete);
 372                         queue_work(bdev->wq, &bo->delayed_delete);
 373                         return;
 374                 }
 375
 376                 ttm_bo_cleanup_memtype_use(bo);
 377                 dma_resv_unlock(bo->base.resv);
 378         }
 379
 380         atomic_dec(&ttm_glob.bo_count);
 381         bo->destroy(bo);
 382 }
 383
 384 /**
 385  * ttm_bo_put
 386  *
 387  * @bo: The buffer object.
 388  *
 389  * Unreference a buffer object.
 390  */
 391 void ttm_bo_put(struct ttm_buffer_object *bo)
 392 {
 393         kref_put(&bo->kref, ttm_bo_release);
 394 }
 395 EXPORT_SYMBOL(ttm_bo_put);
 396
 397 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 398                                      struct ttm_resource **mem,
 399                                      struct ttm_operation_ctx *ctx,
 400                                      struct ttm_place *hop)
 401 {
 402         struct ttm_placement hop_placement;
 403         struct ttm_resource *hop_mem;
 404         int ret;
 405
 406         hop_placement.num_placement = hop_placement.num_busy_placement = 1;
 407         hop_placement.placement = hop_placement.busy_placement = hop;
 408
 409         /* find space in the bounce domain */
 410         ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
 411         if (ret)
 412                 return ret;
 413         /* move to the bounce domain */
 414         ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL);
 415         if (ret) {
 416                 ttm_resource_free(bo, &hop_mem);
 417                 return ret;
 418         }
 419         return 0;
 420 }
 421
 422 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 423                         struct ttm_operation_ctx *ctx)
 424 {
 425         struct ttm_device *bdev = bo->bdev;
 426         struct ttm_resource *evict_mem;
 427         struct ttm_placement placement;
 428         struct ttm_place hop;
 429         int ret = 0;
 430
 431         memset(&hop, 0, sizeof(hop));
 432
 433         dma_resv_assert_held(bo->base.resv);
 434
 435         placement.num_placement = 0;
 436         placement.num_busy_placement = 0;
 437         bdev->funcs->evict_flags(bo, &placement);
 438
 439         if (!placement.num_placement && !placement.num_busy_placement) {
 440                 ret = ttm_bo_wait_ctx(bo, ctx);
 441                 if (ret)
 442                         return ret;
 443
 444                 /*
 445                  * Since we've already synced, this frees backing store
 446                  * immediately.
 447                  */
 448                 return ttm_bo_pipeline_gutting(bo);
 449         }
 450
 451         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 452         if (ret) {
 453                 if (ret != -ERESTARTSYS) {
 454                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 455                                bo);
 456                         ttm_bo_mem_space_debug(bo, &placement);
 457                 }
 458                 goto out;
 459         }
 460
 461 bounce:
 462         ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
 463         if (ret == -EMULTIHOP) {
 464                 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
 465                 if (ret) {
 466                         pr_err("Buffer eviction failed\n");
 467                         ttm_resource_free(bo, &evict_mem);
 468                         goto out;
 469                 }
 470                 /* try and move to final place now. */
 471                 goto bounce;
 472         }
 473 out:
 474         return ret;
 475 }
 476
 477 /**
 478  * ttm_bo_eviction_valuable
 479  *
 480  * @bo: The buffer object to evict
 481  * @place: the placement we need to make room for
 482  *
 483  * Check if it is valuable to evict the BO to make room for the given placement.
 484  */
 485 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 486                               const struct ttm_place *place)
 487 {
 488         struct ttm_resource *res = bo->resource;
 489         struct ttm_device *bdev = bo->bdev;
 490
 491         dma_resv_assert_held(bo->base.resv);
 492         if (bo->resource->mem_type == TTM_PL_SYSTEM)
 493                 return true;
 494
 495         /* Don't evict this BO if it's outside of the
 496          * requested placement range
 497          */
 498         return ttm_resource_intersects(bdev, res, place, bo->base.size);
 499 }
 500 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 501
 502 /*
 503  * Check the target bo is allowable to be evicted or swapout, including cases:
 504  *
 505  * a. if share same reservation object with ctx->resv, have assumption
 506  * reservation objects should already be locked, so not lock again and
 507  * return true directly when either the opreation allow_reserved_eviction
 508  * or the target bo already is in delayed free list;
 509  *
 510  * b. Otherwise, trylock it.
 511  */
 512 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 513                                            struct ttm_operation_ctx *ctx,
 514                                            const struct ttm_place *place,
 515                                            bool *locked, bool *busy)
 516 {
 517         bool ret = false;
 518
 519         if (bo->base.resv == ctx->resv) {
 520                 dma_resv_assert_held(bo->base.resv);
 521                 if (ctx->allow_res_evict)
 522                         ret = true;
 523                 *locked = false;
 524                 if (busy)
 525                         *busy = false;
 526         } else {
 527                 ret = dma_resv_trylock(bo->base.resv);
 528                 *locked = ret;
 529                 if (busy)
 530                         *busy = !ret;
 531         }
 532
 533         if (ret && place && (bo->resource->mem_type != place->mem_type ||
 534                 !bo->bdev->funcs->eviction_valuable(bo, place))) {
 535                 ret = false;
 536                 if (*locked) {
 537                         dma_resv_unlock(bo->base.resv);
 538                         *locked = false;
 539                 }
 540         }
 541
 542         return ret;
 543 }
 544
 545 /**
 546  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 547  *
 548  * @busy_bo: BO which couldn't be locked with trylock
 549  * @ctx: operation context
 550  * @ticket: acquire ticket
 551  *
 552  * Try to lock a busy buffer object to avoid failing eviction.
 553  */
 554 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 555                                    struct ttm_operation_ctx *ctx,
 556                                    struct ww_acquire_ctx *ticket)
 557 {
 558         int r;
 559
 560         if (!busy_bo || !ticket)
 561                 return -EBUSY;
 562
 563         if (ctx->interruptible)
 564                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 565                                                           ticket);
 566         else
 567                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 568
 569         /*
 570          * TODO: It would be better to keep the BO locked until allocation is at
 571          * least tried one more time, but that would mean a much larger rework
 572          * of TTM.
 573          */
 574         if (!r)
 575                 dma_resv_unlock(busy_bo->base.resv);
 576
 577         return r == -EDEADLK ? -EBUSY : r;
 578 }
 579
 580 int ttm_mem_evict_first(struct ttm_device *bdev,
 581                         struct ttm_resource_manager *man,
 582                         const struct ttm_place *place,
 583                         struct ttm_operation_ctx *ctx,
 584                         struct ww_acquire_ctx *ticket)
 585 {
 586         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 587         struct ttm_resource_cursor cursor;
 588         struct ttm_resource *res;
 589         bool locked = false;
 590         int ret;
 591
 592         spin_lock(&bdev->lru_lock);
 593         ttm_resource_manager_for_each_res(man, &cursor, res) {
 594                 bool busy;
 595
 596                 if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
 597                                                     &locked, &busy)) {
 598                         if (busy && !busy_bo && ticket !=
 599                             dma_resv_locking_ctx(res->bo->base.resv))
 600                                 busy_bo = res->bo;
 601                         continue;
 602                 }
 603
 604                 if (ttm_bo_get_unless_zero(res->bo)) {
 605                         bo = res->bo;
 606                         break;
 607                 }
 608                 if (locked)
 609                         dma_resv_unlock(res->bo->base.resv);
 610         }
 611
 612         if (!bo) {
 613                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 614                         busy_bo = NULL;
 615                 spin_unlock(&bdev->lru_lock);
 616                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 617                 if (busy_bo)
 618                         ttm_bo_put(busy_bo);
 619                 return ret;
 620         }
 621
 622         if (bo->deleted) {
 623                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 624                                           ctx->no_wait_gpu, locked);
 625                 ttm_bo_put(bo);
 626                 return ret;
 627         }
 628
 629         spin_unlock(&bdev->lru_lock);
 630
 631         ret = ttm_bo_evict(bo, ctx);
 632         if (locked)
 633                 ttm_bo_unreserve(bo);
 634         else
 635                 ttm_bo_move_to_lru_tail_unlocked(bo);
 636
 637         ttm_bo_put(bo);
 638         return ret;
 639 }
 640
 641 /**
 642  * ttm_bo_pin - Pin the buffer object.
 643  * @bo: The buffer object to pin
 644  *
 645  * Make sure the buffer is not evicted any more during memory pressure.
 646  * @bo must be unpinned again by calling ttm_bo_unpin().
 647  */
 648 void ttm_bo_pin(struct ttm_buffer_object *bo)
 649 {
 650         dma_resv_assert_held(bo->base.resv);
 651         WARN_ON_ONCE(!kref_read(&bo->kref));
 652         spin_lock(&bo->bdev->lru_lock);
 653         if (bo->resource)
 654                 ttm_resource_del_bulk_move(bo->resource, bo);
 655         ++bo->pin_count;
 656         spin_unlock(&bo->bdev->lru_lock);
 657 }
 658 EXPORT_SYMBOL(ttm_bo_pin);
 659
 660 /**
 661  * ttm_bo_unpin - Unpin the buffer object.
 662  * @bo: The buffer object to unpin
 663  *
 664  * Allows the buffer object to be evicted again during memory pressure.
 665  */
 666 void ttm_bo_unpin(struct ttm_buffer_object *bo)
 667 {
 668         dma_resv_assert_held(bo->base.resv);
 669         WARN_ON_ONCE(!kref_read(&bo->kref));
 670         if (WARN_ON_ONCE(!bo->pin_count))
 671                 return;
 672
 673         spin_lock(&bo->bdev->lru_lock);
 674         --bo->pin_count;
 675         if (bo->resource)
 676                 ttm_resource_add_bulk_move(bo->resource, bo);
 677         spin_unlock(&bo->bdev->lru_lock);
 678 }
 679 EXPORT_SYMBOL(ttm_bo_unpin);
 680
 681 /*
 682  * Add the last move fence to the BO as kernel dependency and reserve a new
 683  * fence slot.
 684  */
 685 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 686                                  struct ttm_resource_manager *man,
 687                                  struct ttm_resource *mem,
 688                                  bool no_wait_gpu)
 689 {
 690         struct dma_fence *fence;
 691         int ret;
 692
 693         spin_lock(&man->move_lock);
 694         fence = dma_fence_get(man->move);
 695         spin_unlock(&man->move_lock);
 696
 697         if (!fence)
 698                 return 0;
 699
 700         if (no_wait_gpu) {
 701                 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY;
 702                 dma_fence_put(fence);
 703                 return ret;
 704         }
 705
 706         dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
 707
 708         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 709         dma_fence_put(fence);
 710         return ret;
 711 }
 712
 713 /*
 714  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 715  * space, or we've evicted everything and there isn't enough space.
 716  */
 717 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 718                                   const struct ttm_place *place,
 719                                   struct ttm_resource **mem,
 720                                   struct ttm_operation_ctx *ctx)
 721 {
 722         struct ttm_device *bdev = bo->bdev;
 723         struct ttm_resource_manager *man;
 724         struct ww_acquire_ctx *ticket;
 725         int ret;
 726
 727         man = ttm_manager_type(bdev, place->mem_type);
 728         ticket = dma_resv_locking_ctx(bo->base.resv);
 729         do {
 730                 ret = ttm_resource_alloc(bo, place, mem);
 731                 if (likely(!ret))
 732                         break;
 733                 if (unlikely(ret != -ENOSPC))
 734                         return ret;
 735                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 736                                           ticket);
 737                 if (unlikely(ret != 0))
 738                         return ret;
 739         } while (1);
 740
 741         return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 742 }
 743
 744 /**
 745  * ttm_bo_mem_space
 746  *
 747  * @bo: Pointer to a struct ttm_buffer_object. the data of which
 748  * we want to allocate space for.
 749  * @proposed_placement: Proposed new placement for the buffer object.
 750  * @mem: A struct ttm_resource.
 751  * @ctx: if and how to sleep, lock buffers and alloc memory
 752  *
 753  * Allocate memory space for the buffer object pointed to by @bo, using
 754  * the placement flags in @placement, potentially evicting other idle buffer objects.
 755  * This function may sleep while waiting for space to become available.
 756  * Returns:
 757  * -EBUSY: No space available (only if no_wait == 1).
 758  * -ENOMEM: Could not allocate memory for the buffer object, either due to
 759  * fragmentation or concurrent allocators.
 760  * -ERESTARTSYS: An interruptible sleep was interrupted by a signal.
 761  */
 762 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 763                         struct ttm_placement *placement,
 764                         struct ttm_resource **mem,
 765                         struct ttm_operation_ctx *ctx)
 766 {
 767         struct ttm_device *bdev = bo->bdev;
 768         bool type_found = false;
 769         int i, ret;
 770
 771         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 772         if (unlikely(ret))
 773                 return ret;
 774
 775         for (i = 0; i < placement->num_placement; ++i) {
 776                 const struct ttm_place *place = &placement->placement[i];
 777                 struct ttm_resource_manager *man;
 778
 779                 man = ttm_manager_type(bdev, place->mem_type);
 780                 if (!man || !ttm_resource_manager_used(man))
 781                         continue;
 782
 783                 type_found = true;
 784                 ret = ttm_resource_alloc(bo, place, mem);
 785                 if (ret == -ENOSPC)
 786                         continue;
 787                 if (unlikely(ret))
 788                         goto error;
 789
 790                 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 791                 if (unlikely(ret)) {
 792                         ttm_resource_free(bo, mem);
 793                         if (ret == -EBUSY)
 794                                 continue;
 795
 796                         goto error;
 797                 }
 798                 return 0;
 799         }
 800
 801         for (i = 0; i < placement->num_busy_placement; ++i) {
 802                 const struct ttm_place *place = &placement->busy_placement[i];
 803                 struct ttm_resource_manager *man;
 804
 805                 man = ttm_manager_type(bdev, place->mem_type);
 806                 if (!man || !ttm_resource_manager_used(man))
 807                         continue;
 808
 809                 type_found = true;
 810                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 811                 if (likely(!ret))
 812                         return 0;
 813
 814                 if (ret && ret != -EBUSY)
 815                         goto error;
 816         }
 817
 818         ret = -ENOMEM;
 819         if (!type_found) {
 820                 pr_err(TTM_PFX "No compatible memory type found\n");
 821                 ret = -EINVAL;
 822         }
 823
 824 error:
 825         return ret;
 826 }
 827 EXPORT_SYMBOL(ttm_bo_mem_space);
 828
 829 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 830                               struct ttm_placement *placement,
 831                               struct ttm_operation_ctx *ctx)
 832 {
 833         struct ttm_resource *mem;
 834         struct ttm_place hop;
 835         int ret;
 836
 837         dma_resv_assert_held(bo->base.resv);
 838
 839         /*
 840          * Determine where to move the buffer.
 841          *
 842          * If driver determines move is going to need
 843          * an extra step then it will return -EMULTIHOP
 844          * and the buffer will be moved to the temporary
 845          * stop and the driver will be called to make
 846          * the second hop.
 847          */
 848         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 849         if (ret)
 850                 return ret;
 851 bounce:
 852         ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
 853         if (ret == -EMULTIHOP) {
 854                 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
 855                 if (ret)
 856                         goto out;
 857                 /* try and move to final place now. */
 858                 goto bounce;
 859         }
 860 out:
 861         if (ret)
 862                 ttm_resource_free(bo, &mem);
 863         return ret;
 864 }
 865
 866 /**
 867  * ttm_bo_validate
 868  *
 869  * @bo: The buffer object.
 870  * @placement: Proposed placement for the buffer object.
 871  * @ctx: validation parameters.
 872  *
 873  * Changes placement and caching policy of the buffer object
 874  * according proposed placement.
 875  * Returns
 876  * -EINVAL on invalid proposed placement.
 877  * -ENOMEM on out-of-memory condition.
 878  * -EBUSY if no_wait is true and buffer busy.
 879  * -ERESTARTSYS if interrupted by a signal.
 880  */
 881 int ttm_bo_validate(struct ttm_buffer_object *bo,
 882                     struct ttm_placement *placement,
 883                     struct ttm_operation_ctx *ctx)
 884 {
 885         int ret;
 886
 887         dma_resv_assert_held(bo->base.resv);
 888
 889         /*
 890          * Remove the backing store if no placement is given.
 891          */
 892         if (!placement->num_placement && !placement->num_busy_placement)
 893                 return ttm_bo_pipeline_gutting(bo);
 894
 895         /*
 896          * Check whether we need to move buffer.
 897          */
 898         if (!bo->resource || !ttm_resource_compat(bo->resource, placement)) {
 899                 ret = ttm_bo_move_buffer(bo, placement, ctx);
 900                 if (ret)
 901                         return ret;
 902         }
 903         /*
 904          * We might need to add a TTM.
 905          */
 906         if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) {
 907                 ret = ttm_tt_create(bo, true);
 908                 if (ret)
 909                         return ret;
 910         }
 911         return 0;
 912 }
 913 EXPORT_SYMBOL(ttm_bo_validate);
 914
 915 /**
 916  * ttm_bo_init_reserved
 917  *
 918  * @bdev: Pointer to a ttm_device struct.
 919  * @bo: Pointer to a ttm_buffer_object to be initialized.
 920  * @type: Requested type of buffer object.
 921  * @placement: Initial placement for buffer object.
 922  * @alignment: Data alignment in pages.
 923  * @ctx: TTM operation context for memory allocation.
 924  * @sg: Scatter-gather table.
 925  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
 926  * @destroy: Destroy function. Use NULL for kfree().
 927  *
 928  * This function initializes a pre-allocated struct ttm_buffer_object.
 929  * As this object may be part of a larger structure, this function,
 930  * together with the @destroy function, enables driver-specific objects
 931  * derived from a ttm_buffer_object.
 932  *
 933  * On successful return, the caller owns an object kref to @bo. The kref and
 934  * list_kref are usually set to 1, but note that in some situations, other
 935  * tasks may already be holding references to @bo as well.
 936  * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
 937  * and it is the caller's responsibility to call ttm_bo_unreserve.
 938  *
 939  * If a failure occurs, the function will call the @destroy function. Thus,
 940  * after a failure, dereferencing @bo is illegal and will likely cause memory
 941  * corruption.
 942  *
 943  * Returns
 944  * -ENOMEM: Out of memory.
 945  * -EINVAL: Invalid placement flags.
 946  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
 947  */
 948 int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 949                          enum ttm_bo_type type, struct ttm_placement *placement,
 950                          uint32_t alignment, struct ttm_operation_ctx *ctx,
 951                          struct sg_table *sg, struct dma_resv *resv,
 952                          void (*destroy) (struct ttm_buffer_object *))
 953 {
 954         static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM };
 955         int ret;
 956
 957         kref_init(&bo->kref);
 958         bo->bdev = bdev;
 959         bo->type = type;
 960         bo->page_alignment = alignment;
 961         bo->destroy = destroy;
 962         bo->pin_count = 0;
 963         bo->sg = sg;
 964         bo->bulk_move = NULL;
 965         if (resv)
 966                 bo->base.resv = resv;
 967         else
 968                 bo->base.resv = &bo->base._resv;
 969         atomic_inc(&ttm_glob.bo_count);
 970
 971         ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource);
 972         if (unlikely(ret)) {
 973                 ttm_bo_put(bo);
 974                 return ret;
 975         }
 976
 977         /*
 978          * For ttm_bo_type_device buffers, allocate
 979          * address space from the device.
 980          */
 981         if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) {
 982                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
 983                                          PFN_UP(bo->base.size));
 984                 if (ret)
 985                         goto err_put;
 986         }
 987
 988         /* passed reservation objects should already be locked,
 989          * since otherwise lockdep will be angered in radeon.
 990          */
 991         if (!resv)
 992                 WARN_ON(!dma_resv_trylock(bo->base.resv));
 993         else
 994                 dma_resv_assert_held(resv);
 995
 996         ret = ttm_bo_validate(bo, placement, ctx);
 997         if (unlikely(ret))
 998                 goto err_unlock;
 999
1000         return 0;
1001
1002 err_unlock:
1003         if (!resv)
1004                 dma_resv_unlock(bo->base.resv);
1005
1006 err_put:
1007         ttm_bo_put(bo);
1008         return ret;
1009 }
1010 EXPORT_SYMBOL(ttm_bo_init_reserved);
1011
1012 /**
1013  * ttm_bo_init_validate
1014  *
1015  * @bdev: Pointer to a ttm_device struct.
1016  * @bo: Pointer to a ttm_buffer_object to be initialized.
1017  * @type: Requested type of buffer object.
1018  * @placement: Initial placement for buffer object.
1019  * @alignment: Data alignment in pages.
1020  * @interruptible: If needing to sleep to wait for GPU resources,
1021  * sleep interruptible.
1022  * pinned in physical memory. If this behaviour is not desired, this member
1023  * holds a pointer to a persistent shmem object. Typically, this would
1024  * point to the shmem object backing a GEM object if TTM is used to back a
1025  * GEM user interface.
1026  * @sg: Scatter-gather table.
1027  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
1028  * @destroy: Destroy function. Use NULL for kfree().
1029  *
1030  * This function initializes a pre-allocated struct ttm_buffer_object.
1031  * As this object may be part of a larger structure, this function,
1032  * together with the @destroy function,
1033  * enables driver-specific objects derived from a ttm_buffer_object.
1034  *
1035  * On successful return, the caller owns an object kref to @bo. The kref and
1036  * list_kref are usually set to 1, but note that in some situations, other
1037  * tasks may already be holding references to @bo as well.
1038  *
1039  * If a failure occurs, the function will call the @destroy function, Thus,
1040  * after a failure, dereferencing @bo is illegal and will likely cause memory
1041  * corruption.
1042  *
1043  * Returns
1044  * -ENOMEM: Out of memory.
1045  * -EINVAL: Invalid placement flags.
1046  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
1047  */
1048 int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
1049                          enum ttm_bo_type type, struct ttm_placement *placement,
1050                          uint32_t alignment, bool interruptible,
1051                          struct sg_table *sg, struct dma_resv *resv,
1052                          void (*destroy) (struct ttm_buffer_object *))
1053 {
1054         struct ttm_operation_ctx ctx = { interruptible, false };
1055         int ret;
1056
1057         ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
1058                                    sg, resv, destroy);
1059         if (ret)
1060                 return ret;
1061
1062         if (!resv)
1063                 ttm_bo_unreserve(bo);
1064
1065         return 0;
1066 }
1067 EXPORT_SYMBOL(ttm_bo_init_validate);
1068
1069 /*
1070  * buffer object vm functions.
1071  */
1072
1073 /**
1074  * ttm_bo_unmap_virtual
1075  *
1076  * @bo: tear down the virtual mappings for this BO
1077  */
1078 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1079 {
1080         struct ttm_device *bdev = bo->bdev;
1081
1082         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1083         ttm_mem_io_free(bdev, bo->resource);
1084 }
1085 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1086
1087 /**
1088  * ttm_bo_wait_ctx - wait for buffer idle.
1089  *
1090  * @bo:  The buffer object.
1091  * @ctx: defines how to wait
1092  *
1093  * Waits for the buffer to be idle. Used timeout depends on the context.
1094  * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or
1095  * zero on success.
1096  */
1097 int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx)
1098 {
1099         long ret;
1100
1101         if (ctx->no_wait_gpu) {
1102                 if (dma_resv_test_signaled(bo->base.resv,
1103                                            DMA_RESV_USAGE_BOOKKEEP))
1104                         return 0;
1105                 else
1106                         return -EBUSY;
1107         }
1108
1109         ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1110                                     ctx->interruptible, 15 * HZ);
1111         if (unlikely(ret < 0))
1112                 return ret;
1113         if (unlikely(ret == 0))
1114                 return -EBUSY;
1115         return 0;
1116 }
1117 EXPORT_SYMBOL(ttm_bo_wait_ctx);
1118
1119 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1120                    gfp_t gfp_flags)
1121 {
1122         struct ttm_place place;
1123         bool locked;
1124         long ret;
1125
1126         /*
1127          * While the bo may already reside in SYSTEM placement, set
1128          * SYSTEM as new placement to cover also the move further below.
1129          * The driver may use the fact that we're moving from SYSTEM
1130          * as an indication that we're about to swap out.
1131          */
1132         memset(&place, 0, sizeof(place));
1133         place.mem_type = bo->resource->mem_type;
1134         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
1135                 return -EBUSY;
1136
1137         if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1138             bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
1139             bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED ||
1140             !ttm_bo_get_unless_zero(bo)) {
1141                 if (locked)
1142                         dma_resv_unlock(bo->base.resv);
1143                 return -EBUSY;
1144         }
1145
1146         if (bo->deleted) {
1147                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1148                 ttm_bo_put(bo);
1149                 return ret == -EBUSY ? -ENOSPC : ret;
1150         }
1151
1152         /* TODO: Cleanup the locking */
1153         spin_unlock(&bo->bdev->lru_lock);
1154
1155         /*
1156          * Move to system cached
1157          */
1158         if (bo->resource->mem_type != TTM_PL_SYSTEM) {
1159                 struct ttm_operation_ctx ctx = { false, false };
1160                 struct ttm_resource *evict_mem;
1161                 struct ttm_place hop;
1162
1163                 memset(&hop, 0, sizeof(hop));
1164                 place.mem_type = TTM_PL_SYSTEM;
1165                 ret = ttm_resource_alloc(bo, &place, &evict_mem);
1166                 if (unlikely(ret))
1167                         goto out;
1168
1169                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop);
1170                 if (unlikely(ret != 0)) {
1171                         WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1172                         goto out;
1173                 }
1174         }
1175
1176         /*
1177          * Make sure BO is idle.
1178          */
1179         ret = ttm_bo_wait_ctx(bo, ctx);
1180         if (unlikely(ret != 0))
1181                 goto out;
1182
1183         ttm_bo_unmap_virtual(bo);
1184
1185         /*
1186          * Swap out. Buffer will be swapped in again as soon as
1187          * anyone tries to access a ttm page.
1188          */
1189         if (bo->bdev->funcs->swap_notify)
1190                 bo->bdev->funcs->swap_notify(bo);
1191
1192         if (ttm_tt_is_populated(bo->ttm))
1193                 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1194 out:
1195
1196         /*
1197          * Unreserve without putting on LRU to avoid swapping out an
1198          * already swapped buffer.
1199          */
1200         if (locked)
1201                 dma_resv_unlock(bo->base.resv);
1202         ttm_bo_put(bo);
1203         return ret == -EBUSY ? -ENOSPC : ret;
1204 }
1205
1206 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1207 {
1208         if (bo->ttm == NULL)
1209                 return;
1210
1211         ttm_tt_unpopulate(bo->bdev, bo->ttm);
1212         ttm_tt_destroy(bo->bdev, bo->ttm);
1213         bo->ttm = NULL;
1214 }