drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_bo.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <drm/ttm/ttm_tt.h>
  37
  38 #include <linux/jiffies.h>
  39 #include <linux/slab.h>
  40 #include <linux/sched.h>
  41 #include <linux/mm.h>
  42 #include <linux/file.h>
  43 #include <linux/module.h>
  44 #include <linux/atomic.h>
  45 #include <linux/dma-resv.h>
  46
  47 #include "ttm_module.h"
  48
  49 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  50                                         struct ttm_placement *placement)
  51 {
  52         struct drm_printer p = drm_debug_printer(TTM_PFX);
  53         struct ttm_resource_manager *man;
  54         int i, mem_type;
  55
  56         for (i = 0; i < placement->num_placement; i++) {
  57                 mem_type = placement->placement[i].mem_type;
  58                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  59                            i, placement->placement[i].flags, mem_type);
  60                 man = ttm_manager_type(bo->bdev, mem_type);
  61                 ttm_resource_manager_debug(man, &p);
  62         }
  63 }
  64
  65 /**
  66  * ttm_bo_move_to_lru_tail
  67  *
  68  * @bo: The buffer object.
  69  *
  70  * Move this BO to the tail of all lru lists used to lookup and reserve an
  71  * object. This function must be called with struct ttm_global::lru_lock
  72  * held, and is used to make a BO less likely to be considered for eviction.
  73  */
  74 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
  75 {
  76         dma_resv_assert_held(bo->base.resv);
  77
  78         if (bo->resource)
  79                 ttm_resource_move_to_lru_tail(bo->resource);
  80 }
  81 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
  82
  83 /**
  84  * ttm_bo_set_bulk_move - update BOs bulk move object
  85  *
  86  * @bo: The buffer object.
  87  * @bulk: bulk move structure
  88  *
  89  * Update the BOs bulk move object, making sure that resources are added/removed
  90  * as well. A bulk move allows to move many resource on the LRU at once,
  91  * resulting in much less overhead of maintaining the LRU.
  92  * The only requirement is that the resources stay together on the LRU and are
  93  * never separated. This is enforces by setting the bulk_move structure on a BO.
  94  * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
  95  * their LRU list.
  96  */
  97 void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
  98                           struct ttm_lru_bulk_move *bulk)
  99 {
 100         dma_resv_assert_held(bo->base.resv);
 101
 102         if (bo->bulk_move == bulk)
 103                 return;
 104
 105         spin_lock(&bo->bdev->lru_lock);
 106         if (bo->resource)
 107                 ttm_resource_del_bulk_move(bo->resource, bo);
 108         bo->bulk_move = bulk;
 109         if (bo->resource)
 110                 ttm_resource_add_bulk_move(bo->resource, bo);
 111         spin_unlock(&bo->bdev->lru_lock);
 112 }
 113 EXPORT_SYMBOL(ttm_bo_set_bulk_move);
 114
 115 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 116                                   struct ttm_resource *mem, bool evict,
 117                                   struct ttm_operation_ctx *ctx,
 118                                   struct ttm_place *hop)
 119 {
 120         struct ttm_device *bdev = bo->bdev;
 121         bool old_use_tt, new_use_tt;
 122         int ret;
 123
 124         old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
 125         new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
 126
 127         ttm_bo_unmap_virtual(bo);
 128
 129         /*
 130          * Create and bind a ttm if required.
 131          */
 132
 133         if (new_use_tt) {
 134                 /* Zero init the new TTM structure if the old location should
 135                  * have used one as well.
 136                  */
 137                 ret = ttm_tt_create(bo, old_use_tt);
 138                 if (ret)
 139                         goto out_err;
 140
 141                 if (mem->mem_type != TTM_PL_SYSTEM) {
 142                         ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
 143                         if (ret)
 144                                 goto out_err;
 145                 }
 146         }
 147
 148         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 149         if (ret)
 150                 goto out_err;
 151
 152         ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
 153         if (ret) {
 154                 if (ret == -EMULTIHOP)
 155                         return ret;
 156                 goto out_err;
 157         }
 158
 159         ctx->bytes_moved += bo->base.size;
 160         return 0;
 161
 162 out_err:
 163         if (!old_use_tt)
 164                 ttm_bo_tt_destroy(bo);
 165
 166         return ret;
 167 }
 168
 169 /*
 170  * Call bo::reserved.
 171  * Will release GPU memory type usage on destruction.
 172  * This is the place to put in driver specific hooks to release
 173  * driver private resources.
 174  * Will release the bo::reserved lock.
 175  */
 176
 177 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 178 {
 179         if (bo->bdev->funcs->delete_mem_notify)
 180                 bo->bdev->funcs->delete_mem_notify(bo);
 181
 182         ttm_bo_tt_destroy(bo);
 183         ttm_resource_free(bo, &bo->resource);
 184 }
 185
 186 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 187 {
 188         int r;
 189
 190         if (bo->base.resv == &bo->base._resv)
 191                 return 0;
 192
 193         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 194
 195         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 196         dma_resv_unlock(&bo->base._resv);
 197         if (r)
 198                 return r;
 199
 200         if (bo->type != ttm_bo_type_sg) {
 201                 /* This works because the BO is about to be destroyed and nobody
 202                  * reference it any more. The only tricky case is the trylock on
 203                  * the resv object while holding the lru_lock.
 204                  */
 205                 spin_lock(&bo->bdev->lru_lock);
 206                 bo->base.resv = &bo->base._resv;
 207                 spin_unlock(&bo->bdev->lru_lock);
 208         }
 209
 210         return r;
 211 }
 212
 213 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 214 {
 215         struct dma_resv *resv = &bo->base._resv;
 216         struct dma_resv_iter cursor;
 217         struct dma_fence *fence;
 218
 219         dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
 220         dma_resv_for_each_fence_unlocked(&cursor, fence) {
 221                 if (!fence->ops->signaled)
 222                         dma_fence_enable_sw_signaling(fence);
 223         }
 224         dma_resv_iter_end(&cursor);
 225 }
 226
 227 /**
 228  * ttm_bo_cleanup_refs
 229  * If bo idle, remove from lru lists, and unref.
 230  * If not idle, block if possible.
 231  *
 232  * Must be called with lru_lock and reservation held, this function
 233  * will drop the lru lock and optionally the reservation lock before returning.
 234  *
 235  * @bo:                    The buffer object to clean-up
 236  * @interruptible:         Any sleeps should occur interruptibly.
 237  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
 238  * @unlock_resv:           Unlock the reservation lock as well.
 239  */
 240
 241 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 242                                bool interruptible, bool no_wait_gpu,
 243                                bool unlock_resv)
 244 {
 245         struct dma_resv *resv = &bo->base._resv;
 246         int ret;
 247
 248         if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
 249                 ret = 0;
 250         else
 251                 ret = -EBUSY;
 252
 253         if (ret && !no_wait_gpu) {
 254                 long lret;
 255
 256                 if (unlock_resv)
 257                         dma_resv_unlock(bo->base.resv);
 258                 spin_unlock(&bo->bdev->lru_lock);
 259
 260                 lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
 261                                              interruptible,
 262                                              30 * HZ);
 263
 264                 if (lret < 0)
 265                         return lret;
 266                 else if (lret == 0)
 267                         return -EBUSY;
 268
 269                 spin_lock(&bo->bdev->lru_lock);
 270                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 271                         /*
 272                          * We raced, and lost, someone else holds the reservation now,
 273                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 274                          *
 275                          * Even if it's not the case, because we finished waiting any
 276                          * delayed destruction would succeed, so just return success
 277                          * here.
 278                          */
 279                         spin_unlock(&bo->bdev->lru_lock);
 280                         return 0;
 281                 }
 282                 ret = 0;
 283         }
 284
 285         if (ret) {
 286                 if (unlock_resv)
 287                         dma_resv_unlock(bo->base.resv);
 288                 spin_unlock(&bo->bdev->lru_lock);
 289                 return ret;
 290         }
 291
 292         spin_unlock(&bo->bdev->lru_lock);
 293         ttm_bo_cleanup_memtype_use(bo);
 294
 295         if (unlock_resv)
 296                 dma_resv_unlock(bo->base.resv);
 297
 298         return 0;
 299 }
 300
 301 /*
 302  * Block for the dma_resv object to become idle, lock the buffer and clean up
 303  * the resource and tt object.
 304  */
 305 static void ttm_bo_delayed_delete(struct work_struct *work)
 306 {
 307         struct ttm_buffer_object *bo;
 308
 309         bo = container_of(work, typeof(*bo), delayed_delete);
 310
 311         dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
 312                               MAX_SCHEDULE_TIMEOUT);
 313         dma_resv_lock(bo->base.resv, NULL);
 314         ttm_bo_cleanup_memtype_use(bo);
 315         dma_resv_unlock(bo->base.resv);
 316         ttm_bo_put(bo);
 317 }
 318
 319 static void ttm_bo_release(struct kref *kref)
 320 {
 321         struct ttm_buffer_object *bo =
 322             container_of(kref, struct ttm_buffer_object, kref);
 323         struct ttm_device *bdev = bo->bdev;
 324         int ret;
 325
 326         WARN_ON_ONCE(bo->pin_count);
 327         WARN_ON_ONCE(bo->bulk_move);
 328
 329         if (!bo->deleted) {
 330                 ret = ttm_bo_individualize_resv(bo);
 331                 if (ret) {
 332                         /* Last resort, if we fail to allocate memory for the
 333                          * fences block for the BO to become idle
 334                          */
 335                         dma_resv_wait_timeout(bo->base.resv,
 336                                               DMA_RESV_USAGE_BOOKKEEP, false,
 337                                               30 * HZ);
 338                 }
 339
 340                 if (bo->bdev->funcs->release_notify)
 341                         bo->bdev->funcs->release_notify(bo);
 342
 343                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 344                 ttm_mem_io_free(bdev, bo->resource);
 345
 346                 if (!dma_resv_test_signaled(bo->base.resv,
 347                                             DMA_RESV_USAGE_BOOKKEEP) ||
 348                     (want_init_on_free() && (bo->ttm != NULL)) ||
 349                     !dma_resv_trylock(bo->base.resv)) {
 350                         /* The BO is not idle, resurrect it for delayed destroy */
 351                         ttm_bo_flush_all_fences(bo);
 352                         bo->deleted = true;
 353
 354                         spin_lock(&bo->bdev->lru_lock);
 355
 356                         /*
 357                          * Make pinned bos immediately available to
 358                          * shrinkers, now that they are queued for
 359                          * destruction.
 360                          *
 361                          * FIXME: QXL is triggering this. Can be removed when the
 362                          * driver is fixed.
 363                          */
 364                         if (bo->pin_count) {
 365                                 bo->pin_count = 0;
 366                                 ttm_resource_move_to_lru_tail(bo->resource);
 367                         }
 368
 369                         kref_init(&bo->kref);
 370                         spin_unlock(&bo->bdev->lru_lock);
 371
 372                         INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete);
 373                         queue_work(bdev->wq, &bo->delayed_delete);
 374                         return;
 375                 }
 376
 377                 ttm_bo_cleanup_memtype_use(bo);
 378                 dma_resv_unlock(bo->base.resv);
 379         }
 380
 381         atomic_dec(&ttm_glob.bo_count);
 382         bo->destroy(bo);
 383 }
 384
 385 /**
 386  * ttm_bo_put
 387  *
 388  * @bo: The buffer object.
 389  *
 390  * Unreference a buffer object.
 391  */
 392 void ttm_bo_put(struct ttm_buffer_object *bo)
 393 {
 394         kref_put(&bo->kref, ttm_bo_release);
 395 }
 396 EXPORT_SYMBOL(ttm_bo_put);
 397
 398 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 399                                      struct ttm_resource **mem,
 400                                      struct ttm_operation_ctx *ctx,
 401                                      struct ttm_place *hop)
 402 {
 403         struct ttm_placement hop_placement;
 404         struct ttm_resource *hop_mem;
 405         int ret;
 406
 407         hop_placement.num_placement = hop_placement.num_busy_placement = 1;
 408         hop_placement.placement = hop_placement.busy_placement = hop;
 409
 410         /* find space in the bounce domain */
 411         ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
 412         if (ret)
 413                 return ret;
 414         /* move to the bounce domain */
 415         ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL);
 416         if (ret) {
 417                 ttm_resource_free(bo, &hop_mem);
 418                 return ret;
 419         }
 420         return 0;
 421 }
 422
 423 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 424                         struct ttm_operation_ctx *ctx)
 425 {
 426         struct ttm_device *bdev = bo->bdev;
 427         struct ttm_resource *evict_mem;
 428         struct ttm_placement placement;
 429         struct ttm_place hop;
 430         int ret = 0;
 431
 432         memset(&hop, 0, sizeof(hop));
 433
 434         dma_resv_assert_held(bo->base.resv);
 435
 436         placement.num_placement = 0;
 437         placement.num_busy_placement = 0;
 438         bdev->funcs->evict_flags(bo, &placement);
 439
 440         if (!placement.num_placement && !placement.num_busy_placement) {
 441                 ret = ttm_bo_wait_ctx(bo, ctx);
 442                 if (ret)
 443                         return ret;
 444
 445                 /*
 446                  * Since we've already synced, this frees backing store
 447                  * immediately.
 448                  */
 449                 return ttm_bo_pipeline_gutting(bo);
 450         }
 451
 452         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 453         if (ret) {
 454                 if (ret != -ERESTARTSYS) {
 455                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 456                                bo);
 457                         ttm_bo_mem_space_debug(bo, &placement);
 458                 }
 459                 goto out;
 460         }
 461
 462         do {
 463                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
 464                 if (ret != -EMULTIHOP)
 465                         break;
 466
 467                 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
 468         } while (!ret);
 469
 470         if (ret) {
 471                 ttm_resource_free(bo, &evict_mem);
 472                 if (ret != -ERESTARTSYS && ret != -EINTR)
 473                         pr_err("Buffer eviction failed\n");
 474         }
 475 out:
 476         return ret;
 477 }
 478
 479 /**
 480  * ttm_bo_eviction_valuable
 481  *
 482  * @bo: The buffer object to evict
 483  * @place: the placement we need to make room for
 484  *
 485  * Check if it is valuable to evict the BO to make room for the given placement.
 486  */
 487 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 488                               const struct ttm_place *place)
 489 {
 490         struct ttm_resource *res = bo->resource;
 491         struct ttm_device *bdev = bo->bdev;
 492
 493         dma_resv_assert_held(bo->base.resv);
 494         if (bo->resource->mem_type == TTM_PL_SYSTEM)
 495                 return true;
 496
 497         /* Don't evict this BO if it's outside of the
 498          * requested placement range
 499          */
 500         return ttm_resource_intersects(bdev, res, place, bo->base.size);
 501 }
 502 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 503
 504 /*
 505  * Check the target bo is allowable to be evicted or swapout, including cases:
 506  *
 507  * a. if share same reservation object with ctx->resv, have assumption
 508  * reservation objects should already be locked, so not lock again and
 509  * return true directly when either the opreation allow_reserved_eviction
 510  * or the target bo already is in delayed free list;
 511  *
 512  * b. Otherwise, trylock it.
 513  */
 514 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 515                                            struct ttm_operation_ctx *ctx,
 516                                            const struct ttm_place *place,
 517                                            bool *locked, bool *busy)
 518 {
 519         bool ret = false;
 520
 521         if (bo->pin_count) {
 522                 *locked = false;
 523                 if (busy)
 524                         *busy = false;
 525                 return false;
 526         }
 527
 528         if (bo->base.resv == ctx->resv) {
 529                 dma_resv_assert_held(bo->base.resv);
 530                 if (ctx->allow_res_evict)
 531                         ret = true;
 532                 *locked = false;
 533                 if (busy)
 534                         *busy = false;
 535         } else {
 536                 ret = dma_resv_trylock(bo->base.resv);
 537                 *locked = ret;
 538                 if (busy)
 539                         *busy = !ret;
 540         }
 541
 542         if (ret && place && (bo->resource->mem_type != place->mem_type ||
 543                 !bo->bdev->funcs->eviction_valuable(bo, place))) {
 544                 ret = false;
 545                 if (*locked) {
 546                         dma_resv_unlock(bo->base.resv);
 547                         *locked = false;
 548                 }
 549         }
 550
 551         return ret;
 552 }
 553
 554 /**
 555  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 556  *
 557  * @busy_bo: BO which couldn't be locked with trylock
 558  * @ctx: operation context
 559  * @ticket: acquire ticket
 560  *
 561  * Try to lock a busy buffer object to avoid failing eviction.
 562  */
 563 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 564                                    struct ttm_operation_ctx *ctx,
 565                                    struct ww_acquire_ctx *ticket)
 566 {
 567         int r;
 568
 569         if (!busy_bo || !ticket)
 570                 return -EBUSY;
 571
 572         if (ctx->interruptible)
 573                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 574                                                           ticket);
 575         else
 576                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 577
 578         /*
 579          * TODO: It would be better to keep the BO locked until allocation is at
 580          * least tried one more time, but that would mean a much larger rework
 581          * of TTM.
 582          */
 583         if (!r)
 584                 dma_resv_unlock(busy_bo->base.resv);
 585
 586         return r == -EDEADLK ? -EBUSY : r;
 587 }
 588
 589 int ttm_mem_evict_first(struct ttm_device *bdev,
 590                         struct ttm_resource_manager *man,
 591                         const struct ttm_place *place,
 592                         struct ttm_operation_ctx *ctx,
 593                         struct ww_acquire_ctx *ticket)
 594 {
 595         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 596         struct ttm_resource_cursor cursor;
 597         struct ttm_resource *res;
 598         bool locked = false;
 599         int ret;
 600
 601         spin_lock(&bdev->lru_lock);
 602         ttm_resource_manager_for_each_res(man, &cursor, res) {
 603                 bool busy;
 604
 605                 if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
 606                                                     &locked, &busy)) {
 607                         if (busy && !busy_bo && ticket !=
 608                             dma_resv_locking_ctx(res->bo->base.resv))
 609                                 busy_bo = res->bo;
 610                         continue;
 611                 }
 612
 613                 if (ttm_bo_get_unless_zero(res->bo)) {
 614                         bo = res->bo;
 615                         break;
 616                 }
 617                 if (locked)
 618                         dma_resv_unlock(res->bo->base.resv);
 619         }
 620
 621         if (!bo) {
 622                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 623                         busy_bo = NULL;
 624                 spin_unlock(&bdev->lru_lock);
 625                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 626                 if (busy_bo)
 627                         ttm_bo_put(busy_bo);
 628                 return ret;
 629         }
 630
 631         if (bo->deleted) {
 632                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 633                                           ctx->no_wait_gpu, locked);
 634                 ttm_bo_put(bo);
 635                 return ret;
 636         }
 637
 638         spin_unlock(&bdev->lru_lock);
 639
 640         ret = ttm_bo_evict(bo, ctx);
 641         if (locked)
 642                 ttm_bo_unreserve(bo);
 643         else
 644                 ttm_bo_move_to_lru_tail_unlocked(bo);
 645
 646         ttm_bo_put(bo);
 647         return ret;
 648 }
 649
 650 /**
 651  * ttm_bo_pin - Pin the buffer object.
 652  * @bo: The buffer object to pin
 653  *
 654  * Make sure the buffer is not evicted any more during memory pressure.
 655  * @bo must be unpinned again by calling ttm_bo_unpin().
 656  */
 657 void ttm_bo_pin(struct ttm_buffer_object *bo)
 658 {
 659         dma_resv_assert_held(bo->base.resv);
 660         WARN_ON_ONCE(!kref_read(&bo->kref));
 661         spin_lock(&bo->bdev->lru_lock);
 662         if (bo->resource)
 663                 ttm_resource_del_bulk_move(bo->resource, bo);
 664         ++bo->pin_count;
 665         spin_unlock(&bo->bdev->lru_lock);
 666 }
 667 EXPORT_SYMBOL(ttm_bo_pin);
 668
 669 /**
 670  * ttm_bo_unpin - Unpin the buffer object.
 671  * @bo: The buffer object to unpin
 672  *
 673  * Allows the buffer object to be evicted again during memory pressure.
 674  */
 675 void ttm_bo_unpin(struct ttm_buffer_object *bo)
 676 {
 677         dma_resv_assert_held(bo->base.resv);
 678         WARN_ON_ONCE(!kref_read(&bo->kref));
 679         if (WARN_ON_ONCE(!bo->pin_count))
 680                 return;
 681
 682         spin_lock(&bo->bdev->lru_lock);
 683         --bo->pin_count;
 684         if (bo->resource)
 685                 ttm_resource_add_bulk_move(bo->resource, bo);
 686         spin_unlock(&bo->bdev->lru_lock);
 687 }
 688 EXPORT_SYMBOL(ttm_bo_unpin);
 689
 690 /*
 691  * Add the last move fence to the BO as kernel dependency and reserve a new
 692  * fence slot.
 693  */
 694 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 695                                  struct ttm_resource_manager *man,
 696                                  struct ttm_resource *mem,
 697                                  bool no_wait_gpu)
 698 {
 699         struct dma_fence *fence;
 700         int ret;
 701
 702         spin_lock(&man->move_lock);
 703         fence = dma_fence_get(man->move);
 704         spin_unlock(&man->move_lock);
 705
 706         if (!fence)
 707                 return 0;
 708
 709         if (no_wait_gpu) {
 710                 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY;
 711                 dma_fence_put(fence);
 712                 return ret;
 713         }
 714
 715         dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
 716
 717         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 718         dma_fence_put(fence);
 719         return ret;
 720 }
 721
 722 /*
 723  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 724  * space, or we've evicted everything and there isn't enough space.
 725  */
 726 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 727                                   const struct ttm_place *place,
 728                                   struct ttm_resource **mem,
 729                                   struct ttm_operation_ctx *ctx)
 730 {
 731         struct ttm_device *bdev = bo->bdev;
 732         struct ttm_resource_manager *man;
 733         struct ww_acquire_ctx *ticket;
 734         int ret;
 735
 736         man = ttm_manager_type(bdev, place->mem_type);
 737         ticket = dma_resv_locking_ctx(bo->base.resv);
 738         do {
 739                 ret = ttm_resource_alloc(bo, place, mem);
 740                 if (likely(!ret))
 741                         break;
 742                 if (unlikely(ret != -ENOSPC))
 743                         return ret;
 744                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 745                                           ticket);
 746                 if (unlikely(ret != 0))
 747                         return ret;
 748         } while (1);
 749
 750         return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 751 }
 752
 753 /**
 754  * ttm_bo_mem_space
 755  *
 756  * @bo: Pointer to a struct ttm_buffer_object. the data of which
 757  * we want to allocate space for.
 758  * @placement: Proposed new placement for the buffer object.
 759  * @mem: A struct ttm_resource.
 760  * @ctx: if and how to sleep, lock buffers and alloc memory
 761  *
 762  * Allocate memory space for the buffer object pointed to by @bo, using
 763  * the placement flags in @placement, potentially evicting other idle buffer objects.
 764  * This function may sleep while waiting for space to become available.
 765  * Returns:
 766  * -EBUSY: No space available (only if no_wait == 1).
 767  * -ENOMEM: Could not allocate memory for the buffer object, either due to
 768  * fragmentation or concurrent allocators.
 769  * -ERESTARTSYS: An interruptible sleep was interrupted by a signal.
 770  */
 771 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 772                         struct ttm_placement *placement,
 773                         struct ttm_resource **mem,
 774                         struct ttm_operation_ctx *ctx)
 775 {
 776         struct ttm_device *bdev = bo->bdev;
 777         bool type_found = false;
 778         int i, ret;
 779
 780         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 781         if (unlikely(ret))
 782                 return ret;
 783
 784         for (i = 0; i < placement->num_placement; ++i) {
 785                 const struct ttm_place *place = &placement->placement[i];
 786                 struct ttm_resource_manager *man;
 787
 788                 man = ttm_manager_type(bdev, place->mem_type);
 789                 if (!man || !ttm_resource_manager_used(man))
 790                         continue;
 791
 792                 type_found = true;
 793                 ret = ttm_resource_alloc(bo, place, mem);
 794                 if (ret == -ENOSPC)
 795                         continue;
 796                 if (unlikely(ret))
 797                         goto error;
 798
 799                 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 800                 if (unlikely(ret)) {
 801                         ttm_resource_free(bo, mem);
 802                         if (ret == -EBUSY)
 803                                 continue;
 804
 805                         goto error;
 806                 }
 807                 return 0;
 808         }
 809
 810         for (i = 0; i < placement->num_busy_placement; ++i) {
 811                 const struct ttm_place *place = &placement->busy_placement[i];
 812                 struct ttm_resource_manager *man;
 813
 814                 man = ttm_manager_type(bdev, place->mem_type);
 815                 if (!man || !ttm_resource_manager_used(man))
 816                         continue;
 817
 818                 type_found = true;
 819                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 820                 if (likely(!ret))
 821                         return 0;
 822
 823                 if (ret && ret != -EBUSY)
 824                         goto error;
 825         }
 826
 827         ret = -ENOMEM;
 828         if (!type_found) {
 829                 pr_err(TTM_PFX "No compatible memory type found\n");
 830                 ret = -EINVAL;
 831         }
 832
 833 error:
 834         return ret;
 835 }
 836 EXPORT_SYMBOL(ttm_bo_mem_space);
 837
 838 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 839                               struct ttm_placement *placement,
 840                               struct ttm_operation_ctx *ctx)
 841 {
 842         struct ttm_resource *mem;
 843         struct ttm_place hop;
 844         int ret;
 845
 846         dma_resv_assert_held(bo->base.resv);
 847
 848         /*
 849          * Determine where to move the buffer.
 850          *
 851          * If driver determines move is going to need
 852          * an extra step then it will return -EMULTIHOP
 853          * and the buffer will be moved to the temporary
 854          * stop and the driver will be called to make
 855          * the second hop.
 856          */
 857         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 858         if (ret)
 859                 return ret;
 860 bounce:
 861         ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
 862         if (ret == -EMULTIHOP) {
 863                 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
 864                 if (ret)
 865                         goto out;
 866                 /* try and move to final place now. */
 867                 goto bounce;
 868         }
 869 out:
 870         if (ret)
 871                 ttm_resource_free(bo, &mem);
 872         return ret;
 873 }
 874
 875 /**
 876  * ttm_bo_validate
 877  *
 878  * @bo: The buffer object.
 879  * @placement: Proposed placement for the buffer object.
 880  * @ctx: validation parameters.
 881  *
 882  * Changes placement and caching policy of the buffer object
 883  * according proposed placement.
 884  * Returns
 885  * -EINVAL on invalid proposed placement.
 886  * -ENOMEM on out-of-memory condition.
 887  * -EBUSY if no_wait is true and buffer busy.
 888  * -ERESTARTSYS if interrupted by a signal.
 889  */
 890 int ttm_bo_validate(struct ttm_buffer_object *bo,
 891                     struct ttm_placement *placement,
 892                     struct ttm_operation_ctx *ctx)
 893 {
 894         int ret;
 895
 896         dma_resv_assert_held(bo->base.resv);
 897
 898         /*
 899          * Remove the backing store if no placement is given.
 900          */
 901         if (!placement->num_placement && !placement->num_busy_placement)
 902                 return ttm_bo_pipeline_gutting(bo);
 903
 904         /* Check whether we need to move buffer. */
 905         if (bo->resource && ttm_resource_compat(bo->resource, placement))
 906                 return 0;
 907
 908         /* Moving of pinned BOs is forbidden */
 909         if (bo->pin_count)
 910                 return -EINVAL;
 911
 912         ret = ttm_bo_move_buffer(bo, placement, ctx);
 913         if (ret)
 914                 return ret;
 915
 916         /*
 917          * We might need to add a TTM.
 918          */
 919         if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) {
 920                 ret = ttm_tt_create(bo, true);
 921                 if (ret)
 922                         return ret;
 923         }
 924         return 0;
 925 }
 926 EXPORT_SYMBOL(ttm_bo_validate);
 927
 928 /**
 929  * ttm_bo_init_reserved
 930  *
 931  * @bdev: Pointer to a ttm_device struct.
 932  * @bo: Pointer to a ttm_buffer_object to be initialized.
 933  * @type: Requested type of buffer object.
 934  * @placement: Initial placement for buffer object.
 935  * @alignment: Data alignment in pages.
 936  * @ctx: TTM operation context for memory allocation.
 937  * @sg: Scatter-gather table.
 938  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
 939  * @destroy: Destroy function. Use NULL for kfree().
 940  *
 941  * This function initializes a pre-allocated struct ttm_buffer_object.
 942  * As this object may be part of a larger structure, this function,
 943  * together with the @destroy function, enables driver-specific objects
 944  * derived from a ttm_buffer_object.
 945  *
 946  * On successful return, the caller owns an object kref to @bo. The kref and
 947  * list_kref are usually set to 1, but note that in some situations, other
 948  * tasks may already be holding references to @bo as well.
 949  * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
 950  * and it is the caller's responsibility to call ttm_bo_unreserve.
 951  *
 952  * If a failure occurs, the function will call the @destroy function. Thus,
 953  * after a failure, dereferencing @bo is illegal and will likely cause memory
 954  * corruption.
 955  *
 956  * Returns
 957  * -ENOMEM: Out of memory.
 958  * -EINVAL: Invalid placement flags.
 959  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
 960  */
 961 int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 962                          enum ttm_bo_type type, struct ttm_placement *placement,
 963                          uint32_t alignment, struct ttm_operation_ctx *ctx,
 964                          struct sg_table *sg, struct dma_resv *resv,
 965                          void (*destroy) (struct ttm_buffer_object *))
 966 {
 967         int ret;
 968
 969         kref_init(&bo->kref);
 970         bo->bdev = bdev;
 971         bo->type = type;
 972         bo->page_alignment = alignment;
 973         bo->destroy = destroy;
 974         bo->pin_count = 0;
 975         bo->sg = sg;
 976         bo->bulk_move = NULL;
 977         if (resv)
 978                 bo->base.resv = resv;
 979         else
 980                 bo->base.resv = &bo->base._resv;
 981         atomic_inc(&ttm_glob.bo_count);
 982
 983         /*
 984          * For ttm_bo_type_device buffers, allocate
 985          * address space from the device.
 986          */
 987         if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) {
 988                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
 989                                          PFN_UP(bo->base.size));
 990                 if (ret)
 991                         goto err_put;
 992         }
 993
 994         /* passed reservation objects should already be locked,
 995          * since otherwise lockdep will be angered in radeon.
 996          */
 997         if (!resv)
 998                 WARN_ON(!dma_resv_trylock(bo->base.resv));
 999         else
1000                 dma_resv_assert_held(resv);
1001
1002         ret = ttm_bo_validate(bo, placement, ctx);
1003         if (unlikely(ret))
1004                 goto err_unlock;
1005
1006         return 0;
1007
1008 err_unlock:
1009         if (!resv)
1010                 dma_resv_unlock(bo->base.resv);
1011
1012 err_put:
1013         ttm_bo_put(bo);
1014         return ret;
1015 }
1016 EXPORT_SYMBOL(ttm_bo_init_reserved);
1017
1018 /**
1019  * ttm_bo_init_validate
1020  *
1021  * @bdev: Pointer to a ttm_device struct.
1022  * @bo: Pointer to a ttm_buffer_object to be initialized.
1023  * @type: Requested type of buffer object.
1024  * @placement: Initial placement for buffer object.
1025  * @alignment: Data alignment in pages.
1026  * @interruptible: If needing to sleep to wait for GPU resources,
1027  * sleep interruptible.
1028  * pinned in physical memory. If this behaviour is not desired, this member
1029  * holds a pointer to a persistent shmem object. Typically, this would
1030  * point to the shmem object backing a GEM object if TTM is used to back a
1031  * GEM user interface.
1032  * @sg: Scatter-gather table.
1033  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
1034  * @destroy: Destroy function. Use NULL for kfree().
1035  *
1036  * This function initializes a pre-allocated struct ttm_buffer_object.
1037  * As this object may be part of a larger structure, this function,
1038  * together with the @destroy function,
1039  * enables driver-specific objects derived from a ttm_buffer_object.
1040  *
1041  * On successful return, the caller owns an object kref to @bo. The kref and
1042  * list_kref are usually set to 1, but note that in some situations, other
1043  * tasks may already be holding references to @bo as well.
1044  *
1045  * If a failure occurs, the function will call the @destroy function, Thus,
1046  * after a failure, dereferencing @bo is illegal and will likely cause memory
1047  * corruption.
1048  *
1049  * Returns
1050  * -ENOMEM: Out of memory.
1051  * -EINVAL: Invalid placement flags.
1052  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
1053  */
1054 int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
1055                          enum ttm_bo_type type, struct ttm_placement *placement,
1056                          uint32_t alignment, bool interruptible,
1057                          struct sg_table *sg, struct dma_resv *resv,
1058                          void (*destroy) (struct ttm_buffer_object *))
1059 {
1060         struct ttm_operation_ctx ctx = { interruptible, false };
1061         int ret;
1062
1063         ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
1064                                    sg, resv, destroy);
1065         if (ret)
1066                 return ret;
1067
1068         if (!resv)
1069                 ttm_bo_unreserve(bo);
1070
1071         return 0;
1072 }
1073 EXPORT_SYMBOL(ttm_bo_init_validate);
1074
1075 /*
1076  * buffer object vm functions.
1077  */
1078
1079 /**
1080  * ttm_bo_unmap_virtual
1081  *
1082  * @bo: tear down the virtual mappings for this BO
1083  */
1084 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1085 {
1086         struct ttm_device *bdev = bo->bdev;
1087
1088         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1089         ttm_mem_io_free(bdev, bo->resource);
1090 }
1091 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1092
1093 /**
1094  * ttm_bo_wait_ctx - wait for buffer idle.
1095  *
1096  * @bo:  The buffer object.
1097  * @ctx: defines how to wait
1098  *
1099  * Waits for the buffer to be idle. Used timeout depends on the context.
1100  * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or
1101  * zero on success.
1102  */
1103 int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx)
1104 {
1105         long ret;
1106
1107         if (ctx->no_wait_gpu) {
1108                 if (dma_resv_test_signaled(bo->base.resv,
1109                                            DMA_RESV_USAGE_BOOKKEEP))
1110                         return 0;
1111                 else
1112                         return -EBUSY;
1113         }
1114
1115         ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1116                                     ctx->interruptible, 15 * HZ);
1117         if (unlikely(ret < 0))
1118                 return ret;
1119         if (unlikely(ret == 0))
1120                 return -EBUSY;
1121         return 0;
1122 }
1123 EXPORT_SYMBOL(ttm_bo_wait_ctx);
1124
1125 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1126                    gfp_t gfp_flags)
1127 {
1128         struct ttm_place place;
1129         bool locked;
1130         long ret;
1131
1132         /*
1133          * While the bo may already reside in SYSTEM placement, set
1134          * SYSTEM as new placement to cover also the move further below.
1135          * The driver may use the fact that we're moving from SYSTEM
1136          * as an indication that we're about to swap out.
1137          */
1138         memset(&place, 0, sizeof(place));
1139         place.mem_type = bo->resource->mem_type;
1140         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
1141                 return -EBUSY;
1142
1143         if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1144             bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
1145             bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED ||
1146             !ttm_bo_get_unless_zero(bo)) {
1147                 if (locked)
1148                         dma_resv_unlock(bo->base.resv);
1149                 return -EBUSY;
1150         }
1151
1152         if (bo->deleted) {
1153                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1154                 ttm_bo_put(bo);
1155                 return ret == -EBUSY ? -ENOSPC : ret;
1156         }
1157
1158         /* TODO: Cleanup the locking */
1159         spin_unlock(&bo->bdev->lru_lock);
1160
1161         /*
1162          * Move to system cached
1163          */
1164         if (bo->resource->mem_type != TTM_PL_SYSTEM) {
1165                 struct ttm_resource *evict_mem;
1166                 struct ttm_place hop;
1167
1168                 memset(&hop, 0, sizeof(hop));
1169                 place.mem_type = TTM_PL_SYSTEM;
1170                 ret = ttm_resource_alloc(bo, &place, &evict_mem);
1171                 if (unlikely(ret))
1172                         goto out;
1173
1174                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
1175                 if (unlikely(ret != 0)) {
1176                         WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1177                         ttm_resource_free(bo, &evict_mem);
1178                         goto out;
1179                 }
1180         }
1181
1182         /*
1183          * Make sure BO is idle.
1184          */
1185         ret = ttm_bo_wait_ctx(bo, ctx);
1186         if (unlikely(ret != 0))
1187                 goto out;
1188
1189         ttm_bo_unmap_virtual(bo);
1190
1191         /*
1192          * Swap out. Buffer will be swapped in again as soon as
1193          * anyone tries to access a ttm page.
1194          */
1195         if (bo->bdev->funcs->swap_notify)
1196                 bo->bdev->funcs->swap_notify(bo);
1197
1198         if (ttm_tt_is_populated(bo->ttm))
1199                 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1200 out:
1201
1202         /*
1203          * Unreserve without putting on LRU to avoid swapping out an
1204          * already swapped buffer.
1205          */
1206         if (locked)
1207                 dma_resv_unlock(bo->base.resv);
1208         ttm_bo_put(bo);
1209         return ret == -EBUSY ? -ENOSPC : ret;
1210 }
1211
1212 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1213 {
1214         if (bo->ttm == NULL)
1215                 return;
1216
1217         ttm_tt_unpopulate(bo->bdev, bo->ttm);
1218         ttm_tt_destroy(bo->bdev, bo->ttm);
1219         bo->ttm = NULL;
1220 }