drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /**************************************************************************
   3  *
   4  * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "vmwgfx_bo.h"
  29 #include "vmwgfx_drv.h"
  30 #include <drm/ttm/ttm_placement.h>
  31
  32 static const struct ttm_place vram_placement_flags = {
  33         .fpfn = 0,
  34         .lpfn = 0,
  35         .mem_type = TTM_PL_VRAM,
  36         .flags = 0
  37 };
  38
  39 static const struct ttm_place sys_placement_flags = {
  40         .fpfn = 0,
  41         .lpfn = 0,
  42         .mem_type = TTM_PL_SYSTEM,
  43         .flags = 0
  44 };
  45
  46 static const struct ttm_place gmr_placement_flags = {
  47         .fpfn = 0,
  48         .lpfn = 0,
  49         .mem_type = VMW_PL_GMR,
  50         .flags = 0
  51 };
  52
  53 struct ttm_placement vmw_vram_placement = {
  54         .num_placement = 1,
  55         .placement = &vram_placement_flags,
  56         .num_busy_placement = 1,
  57         .busy_placement = &vram_placement_flags
  58 };
  59
  60 static const struct ttm_place vram_gmr_placement_flags[] = {
  61         {
  62                 .fpfn = 0,
  63                 .lpfn = 0,
  64                 .mem_type = TTM_PL_VRAM,
  65                 .flags = 0
  66         }, {
  67                 .fpfn = 0,
  68                 .lpfn = 0,
  69                 .mem_type = VMW_PL_GMR,
  70                 .flags = 0
  71         }
  72 };
  73
  74 struct ttm_placement vmw_vram_gmr_placement = {
  75         .num_placement = 2,
  76         .placement = vram_gmr_placement_flags,
  77         .num_busy_placement = 1,
  78         .busy_placement = &gmr_placement_flags
  79 };
  80
  81 struct ttm_placement vmw_sys_placement = {
  82         .num_placement = 1,
  83         .placement = &sys_placement_flags,
  84         .num_busy_placement = 1,
  85         .busy_placement = &sys_placement_flags
  86 };
  87
  88 const size_t vmw_tt_size = sizeof(struct vmw_ttm_tt);
  89
  90 /**
  91  * __vmw_piter_non_sg_next: Helper functions to advance
  92  * a struct vmw_piter iterator.
  93  *
  94  * @viter: Pointer to the iterator.
  95  *
  96  * These functions return false if past the end of the list,
  97  * true otherwise. Functions are selected depending on the current
  98  * DMA mapping mode.
  99  */
 100 static bool __vmw_piter_non_sg_next(struct vmw_piter *viter)
 101 {
 102         return ++(viter->i) < viter->num_pages;
 103 }
 104
 105 static bool __vmw_piter_sg_next(struct vmw_piter *viter)
 106 {
 107         bool ret = __vmw_piter_non_sg_next(viter);
 108
 109         return __sg_page_iter_dma_next(&viter->iter) && ret;
 110 }
 111
 112
 113 static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter)
 114 {
 115         return viter->addrs[viter->i];
 116 }
 117
 118 static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter)
 119 {
 120         return sg_page_iter_dma_address(&viter->iter);
 121 }
 122
 123
 124 /**
 125  * vmw_piter_start - Initialize a struct vmw_piter.
 126  *
 127  * @viter: Pointer to the iterator to initialize
 128  * @vsgt: Pointer to a struct vmw_sg_table to initialize from
 129  * @p_offset: Pointer offset used to update current array position
 130  *
 131  * Note that we're following the convention of __sg_page_iter_start, so that
 132  * the iterator doesn't point to a valid page after initialization; it has
 133  * to be advanced one step first.
 134  */
 135 void vmw_piter_start(struct vmw_piter *viter, const struct vmw_sg_table *vsgt,
 136                      unsigned long p_offset)
 137 {
 138         viter->i = p_offset - 1;
 139         viter->num_pages = vsgt->num_pages;
 140         viter->pages = vsgt->pages;
 141         switch (vsgt->mode) {
 142         case vmw_dma_alloc_coherent:
 143                 viter->next = &__vmw_piter_non_sg_next;
 144                 viter->dma_address = &__vmw_piter_dma_addr;
 145                 viter->addrs = vsgt->addrs;
 146                 break;
 147         case vmw_dma_map_populate:
 148         case vmw_dma_map_bind:
 149                 viter->next = &__vmw_piter_sg_next;
 150                 viter->dma_address = &__vmw_piter_sg_addr;
 151                 __sg_page_iter_start(&viter->iter.base, vsgt->sgt->sgl,
 152                                      vsgt->sgt->orig_nents, p_offset);
 153                 break;
 154         default:
 155                 BUG();
 156         }
 157 }
 158
 159 /**
 160  * vmw_ttm_unmap_from_dma - unmap  device addresses previsouly mapped for
 161  * TTM pages
 162  *
 163  * @vmw_tt: Pointer to a struct vmw_ttm_backend
 164  *
 165  * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
 166  */
 167 static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt *vmw_tt)
 168 {
 169         struct device *dev = vmw_tt->dev_priv->drm.dev;
 170
 171         dma_unmap_sgtable(dev, &vmw_tt->sgt, DMA_BIDIRECTIONAL, 0);
 172         vmw_tt->sgt.nents = vmw_tt->sgt.orig_nents;
 173 }
 174
 175 /**
 176  * vmw_ttm_map_for_dma - map TTM pages to get device addresses
 177  *
 178  * @vmw_tt: Pointer to a struct vmw_ttm_backend
 179  *
 180  * This function is used to get device addresses from the kernel DMA layer.
 181  * However, it's violating the DMA API in that when this operation has been
 182  * performed, it's illegal for the CPU to write to the pages without first
 183  * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
 184  * therefore only legal to call this function if we know that the function
 185  * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
 186  * a CPU write buffer flush.
 187  */
 188 static int vmw_ttm_map_for_dma(struct vmw_ttm_tt *vmw_tt)
 189 {
 190         struct device *dev = vmw_tt->dev_priv->drm.dev;
 191
 192         return dma_map_sgtable(dev, &vmw_tt->sgt, DMA_BIDIRECTIONAL, 0);
 193 }
 194
 195 /**
 196  * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
 197  *
 198  * @vmw_tt: Pointer to a struct vmw_ttm_tt
 199  *
 200  * Select the correct function for and make sure the TTM pages are
 201  * visible to the device. Allocate storage for the device mappings.
 202  * If a mapping has already been performed, indicated by the storage
 203  * pointer being non NULL, the function returns success.
 204  */
 205 static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
 206 {
 207         struct vmw_private *dev_priv = vmw_tt->dev_priv;
 208         struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
 209         int ret = 0;
 210
 211         if (vmw_tt->mapped)
 212                 return 0;
 213
 214         vsgt->mode = dev_priv->map_mode;
 215         vsgt->pages = vmw_tt->dma_ttm.pages;
 216         vsgt->num_pages = vmw_tt->dma_ttm.num_pages;
 217         vsgt->addrs = vmw_tt->dma_ttm.dma_address;
 218         vsgt->sgt = NULL;
 219
 220         switch (dev_priv->map_mode) {
 221         case vmw_dma_map_bind:
 222         case vmw_dma_map_populate:
 223                 vsgt->sgt = &vmw_tt->sgt;
 224                 ret = sg_alloc_table_from_pages_segment(
 225                         &vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0,
 226                         (unsigned long)vsgt->num_pages << PAGE_SHIFT,
 227                         dma_get_max_seg_size(dev_priv->drm.dev), GFP_KERNEL);
 228                 if (ret)
 229                         goto out_sg_alloc_fail;
 230
 231                 ret = vmw_ttm_map_for_dma(vmw_tt);
 232                 if (unlikely(ret != 0))
 233                         goto out_map_fail;
 234
 235                 break;
 236         default:
 237                 break;
 238         }
 239
 240         vmw_tt->mapped = true;
 241         return 0;
 242
 243 out_map_fail:
 244         sg_free_table(vmw_tt->vsgt.sgt);
 245         vmw_tt->vsgt.sgt = NULL;
 246 out_sg_alloc_fail:
 247         return ret;
 248 }
 249
 250 /**
 251  * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
 252  *
 253  * @vmw_tt: Pointer to a struct vmw_ttm_tt
 254  *
 255  * Tear down any previously set up device DMA mappings and free
 256  * any storage space allocated for them. If there are no mappings set up,
 257  * this function is a NOP.
 258  */
 259 static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt)
 260 {
 261         struct vmw_private *dev_priv = vmw_tt->dev_priv;
 262
 263         if (!vmw_tt->vsgt.sgt)
 264                 return;
 265
 266         switch (dev_priv->map_mode) {
 267         case vmw_dma_map_bind:
 268         case vmw_dma_map_populate:
 269                 vmw_ttm_unmap_from_dma(vmw_tt);
 270                 sg_free_table(vmw_tt->vsgt.sgt);
 271                 vmw_tt->vsgt.sgt = NULL;
 272                 break;
 273         default:
 274                 break;
 275         }
 276         vmw_tt->mapped = false;
 277 }
 278
 279 /**
 280  * vmw_bo_sg_table - Return a struct vmw_sg_table object for a
 281  * TTM buffer object
 282  *
 283  * @bo: Pointer to a struct ttm_buffer_object
 284  *
 285  * Returns a pointer to a struct vmw_sg_table object. The object should
 286  * not be freed after use.
 287  * Note that for the device addresses to be valid, the buffer object must
 288  * either be reserved or pinned.
 289  */
 290 const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo)
 291 {
 292         struct vmw_ttm_tt *vmw_tt =
 293                 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm);
 294
 295         return &vmw_tt->vsgt;
 296 }
 297
 298
 299 static int vmw_ttm_bind(struct ttm_device *bdev,
 300                         struct ttm_tt *ttm, struct ttm_resource *bo_mem)
 301 {
 302         struct vmw_ttm_tt *vmw_be =
 303                 container_of(ttm, struct vmw_ttm_tt, dma_ttm);
 304         int ret = 0;
 305
 306         if (!bo_mem)
 307                 return -EINVAL;
 308
 309         if (vmw_be->bound)
 310                 return 0;
 311
 312         ret = vmw_ttm_map_dma(vmw_be);
 313         if (unlikely(ret != 0))
 314                 return ret;
 315
 316         vmw_be->gmr_id = bo_mem->start;
 317         vmw_be->mem_type = bo_mem->mem_type;
 318
 319         switch (bo_mem->mem_type) {
 320         case VMW_PL_GMR:
 321                 ret = vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt,
 322                                     ttm->num_pages, vmw_be->gmr_id);
 323                 break;
 324         case VMW_PL_MOB:
 325                 if (unlikely(vmw_be->mob == NULL)) {
 326                         vmw_be->mob =
 327                                 vmw_mob_create(ttm->num_pages);
 328                         if (unlikely(vmw_be->mob == NULL))
 329                                 return -ENOMEM;
 330                 }
 331
 332                 ret = vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob,
 333                                     &vmw_be->vsgt, ttm->num_pages,
 334                                     vmw_be->gmr_id);
 335                 break;
 336         case VMW_PL_SYSTEM:
 337                 /* Nothing to be done for a system bind */
 338                 break;
 339         default:
 340                 BUG();
 341         }
 342         vmw_be->bound = true;
 343         return ret;
 344 }
 345
 346 static void vmw_ttm_unbind(struct ttm_device *bdev,
 347                            struct ttm_tt *ttm)
 348 {
 349         struct vmw_ttm_tt *vmw_be =
 350                 container_of(ttm, struct vmw_ttm_tt, dma_ttm);
 351
 352         if (!vmw_be->bound)
 353                 return;
 354
 355         switch (vmw_be->mem_type) {
 356         case VMW_PL_GMR:
 357                 vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id);
 358                 break;
 359         case VMW_PL_MOB:
 360                 vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob);
 361                 break;
 362         case VMW_PL_SYSTEM:
 363                 break;
 364         default:
 365                 BUG();
 366         }
 367
 368         if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind)
 369                 vmw_ttm_unmap_dma(vmw_be);
 370         vmw_be->bound = false;
 371 }
 372
 373
 374 static void vmw_ttm_destroy(struct ttm_device *bdev, struct ttm_tt *ttm)
 375 {
 376         struct vmw_ttm_tt *vmw_be =
 377                 container_of(ttm, struct vmw_ttm_tt, dma_ttm);
 378
 379         vmw_ttm_unmap_dma(vmw_be);
 380         ttm_tt_fini(ttm);
 381         if (vmw_be->mob)
 382                 vmw_mob_destroy(vmw_be->mob);
 383
 384         kfree(vmw_be);
 385 }
 386
 387
 388 static int vmw_ttm_populate(struct ttm_device *bdev,
 389                             struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
 390 {
 391         int ret;
 392
 393         /* TODO: maybe completely drop this ? */
 394         if (ttm_tt_is_populated(ttm))
 395                 return 0;
 396
 397         ret = ttm_pool_alloc(&bdev->pool, ttm, ctx);
 398
 399         return ret;
 400 }
 401
 402 static void vmw_ttm_unpopulate(struct ttm_device *bdev,
 403                                struct ttm_tt *ttm)
 404 {
 405         struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt,
 406                                                  dma_ttm);
 407
 408         vmw_ttm_unbind(bdev, ttm);
 409
 410         if (vmw_tt->mob) {
 411                 vmw_mob_destroy(vmw_tt->mob);
 412                 vmw_tt->mob = NULL;
 413         }
 414
 415         vmw_ttm_unmap_dma(vmw_tt);
 416
 417         ttm_pool_free(&bdev->pool, ttm);
 418 }
 419
 420 static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo,
 421                                         uint32_t page_flags)
 422 {
 423         struct vmw_ttm_tt *vmw_be;
 424         int ret;
 425
 426         vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL);
 427         if (!vmw_be)
 428                 return NULL;
 429
 430         vmw_be->dev_priv = vmw_priv_from_ttm(bo->bdev);
 431         vmw_be->mob = NULL;
 432
 433         if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
 434                 ret = ttm_sg_tt_init(&vmw_be->dma_ttm, bo, page_flags,
 435                                      ttm_cached);
 436         else
 437                 ret = ttm_tt_init(&vmw_be->dma_ttm, bo, page_flags,
 438                                   ttm_cached, 0);
 439         if (unlikely(ret != 0))
 440                 goto out_no_init;
 441
 442         return &vmw_be->dma_ttm;
 443 out_no_init:
 444         kfree(vmw_be);
 445         return NULL;
 446 }
 447
 448 static void vmw_evict_flags(struct ttm_buffer_object *bo,
 449                      struct ttm_placement *placement)
 450 {
 451         *placement = vmw_sys_placement;
 452 }
 453
 454 static int vmw_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem)
 455 {
 456         struct vmw_private *dev_priv = vmw_priv_from_ttm(bdev);
 457
 458         switch (mem->mem_type) {
 459         case TTM_PL_SYSTEM:
 460         case VMW_PL_SYSTEM:
 461         case VMW_PL_GMR:
 462         case VMW_PL_MOB:
 463                 return 0;
 464         case TTM_PL_VRAM:
 465                 mem->bus.offset = (mem->start << PAGE_SHIFT) +
 466                         dev_priv->vram_start;
 467                 mem->bus.is_iomem = true;
 468                 mem->bus.caching = ttm_cached;
 469                 break;
 470         default:
 471                 return -EINVAL;
 472         }
 473         return 0;
 474 }
 475
 476 /**
 477  * vmw_move_notify - TTM move_notify_callback
 478  *
 479  * @bo: The TTM buffer object about to move.
 480  * @old_mem: The old memory where we move from
 481  * @new_mem: The struct ttm_resource indicating to what memory
 482  *       region the move is taking place.
 483  *
 484  * Calls move_notify for all subsystems needing it.
 485  * (currently only resources).
 486  */
 487 static void vmw_move_notify(struct ttm_buffer_object *bo,
 488                             struct ttm_resource *old_mem,
 489                             struct ttm_resource *new_mem)
 490 {
 491         vmw_bo_move_notify(bo, new_mem);
 492         vmw_query_move_notify(bo, old_mem, new_mem);
 493 }
 494
 495
 496 /**
 497  * vmw_swap_notify - TTM move_notify_callback
 498  *
 499  * @bo: The TTM buffer object about to be swapped out.
 500  */
 501 static void vmw_swap_notify(struct ttm_buffer_object *bo)
 502 {
 503         vmw_bo_swap_notify(bo);
 504         (void) ttm_bo_wait(bo, false, false);
 505 }
 506
 507 static bool vmw_memtype_is_system(uint32_t mem_type)
 508 {
 509         return mem_type == TTM_PL_SYSTEM || mem_type == VMW_PL_SYSTEM;
 510 }
 511
 512 static int vmw_move(struct ttm_buffer_object *bo,
 513                     bool evict,
 514                     struct ttm_operation_ctx *ctx,
 515                     struct ttm_resource *new_mem,
 516                     struct ttm_place *hop)
 517 {
 518         struct ttm_resource_manager *new_man;
 519         struct ttm_resource_manager *old_man = NULL;
 520         int ret = 0;
 521
 522         new_man = ttm_manager_type(bo->bdev, new_mem->mem_type);
 523         if (bo->resource)
 524                 old_man = ttm_manager_type(bo->bdev, bo->resource->mem_type);
 525
 526         if (new_man->use_tt && !vmw_memtype_is_system(new_mem->mem_type)) {
 527                 ret = vmw_ttm_bind(bo->bdev, bo->ttm, new_mem);
 528                 if (ret)
 529                         return ret;
 530         }
 531
 532         if (!bo->resource || (bo->resource->mem_type == TTM_PL_SYSTEM &&
 533                               bo->ttm == NULL)) {
 534                 ttm_bo_move_null(bo, new_mem);
 535                 return 0;
 536         }
 537
 538         vmw_move_notify(bo, bo->resource, new_mem);
 539
 540         if (old_man && old_man->use_tt && new_man->use_tt) {
 541                 if (vmw_memtype_is_system(bo->resource->mem_type)) {
 542                         ttm_bo_move_null(bo, new_mem);
 543                         return 0;
 544                 }
 545                 ret = ttm_bo_wait_ctx(bo, ctx);
 546                 if (ret)
 547                         goto fail;
 548
 549                 vmw_ttm_unbind(bo->bdev, bo->ttm);
 550                 ttm_resource_free(bo, &bo->resource);
 551                 ttm_bo_assign_mem(bo, new_mem);
 552                 return 0;
 553         } else {
 554                 ret = ttm_bo_move_memcpy(bo, ctx, new_mem);
 555                 if (ret)
 556                         goto fail;
 557         }
 558         return 0;
 559 fail:
 560         vmw_move_notify(bo, new_mem, bo->resource);
 561         return ret;
 562 }
 563
 564 struct ttm_device_funcs vmw_bo_driver = {
 565         .ttm_tt_create = &vmw_ttm_tt_create,
 566         .ttm_tt_populate = &vmw_ttm_populate,
 567         .ttm_tt_unpopulate = &vmw_ttm_unpopulate,
 568         .ttm_tt_destroy = &vmw_ttm_destroy,
 569         .eviction_valuable = ttm_bo_eviction_valuable,
 570         .evict_flags = vmw_evict_flags,
 571         .move = vmw_move,
 572         .swap_notify = vmw_swap_notify,
 573         .io_mem_reserve = &vmw_ttm_io_mem_reserve,
 574 };
 575
 576 int vmw_bo_create_and_populate(struct vmw_private *dev_priv,
 577                                size_t bo_size, u32 domain,
 578                                struct vmw_bo **bo_p)
 579 {
 580         struct ttm_operation_ctx ctx = {
 581                 .interruptible = false,
 582                 .no_wait_gpu = false
 583         };
 584         struct vmw_bo *vbo;
 585         int ret;
 586         struct vmw_bo_params bo_params = {
 587                 .domain = domain,
 588                 .busy_domain = domain,
 589                 .bo_type = ttm_bo_type_kernel,
 590                 .size = bo_size,
 591                 .pin = true
 592         };
 593
 594         ret = vmw_bo_create(dev_priv, &bo_params, &vbo);
 595         if (unlikely(ret != 0))
 596                 return ret;
 597
 598         ret = ttm_bo_reserve(&vbo->tbo, false, true, NULL);
 599         BUG_ON(ret != 0);
 600         ret = vmw_ttm_populate(vbo->tbo.bdev, vbo->tbo.ttm, &ctx);
 601         if (likely(ret == 0)) {
 602                 struct vmw_ttm_tt *vmw_tt =
 603                         container_of(vbo->tbo.ttm, struct vmw_ttm_tt, dma_ttm);
 604                 ret = vmw_ttm_map_dma(vmw_tt);
 605         }
 606
 607         ttm_bo_unreserve(&vbo->tbo);
 608
 609         if (likely(ret == 0))
 610                 *bo_p = vbo;
 611         return ret;
 612 }