drivers/gpu/drm/ttm/ttm_resource.c

   1 /*
   2  * Copyright 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Christian König
  23  */
  24
  25 #include <linux/iosys-map.h>
  26 #include <linux/io-mapping.h>
  27 #include <linux/scatterlist.h>
  28
  29 #include <drm/ttm/ttm_bo.h>
  30 #include <drm/ttm/ttm_placement.h>
  31 #include <drm/ttm/ttm_resource.h>
  32
  33 /**
  34  * ttm_lru_bulk_move_init - initialize a bulk move structure
  35  * @bulk: the structure to init
  36  *
  37  * For now just memset the structure to zero.
  38  */
  39 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
  40 {
  41         memset(bulk, 0, sizeof(*bulk));
  42 }
  43 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
  44
  45 /**
  46  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
  47  *
  48  * @bulk: bulk move structure
  49  *
  50  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
  51  * resource order never changes. Should be called with &ttm_device.lru_lock held.
  52  */
  53 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
  54 {
  55         unsigned i, j;
  56
  57         for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
  58                 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
  59                         struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
  60                         struct ttm_resource_manager *man;
  61
  62                         if (!pos->first)
  63                                 continue;
  64
  65                         lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
  66                         dma_resv_assert_held(pos->first->bo->base.resv);
  67                         dma_resv_assert_held(pos->last->bo->base.resv);
  68
  69                         man = ttm_manager_type(pos->first->bo->bdev, i);
  70                         list_bulk_move_tail(&man->lru[j], &pos->first->lru,
  71                                             &pos->last->lru);
  72                 }
  73         }
  74 }
  75 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
  76
  77 /* Return the bulk move pos object for this resource */
  78 static struct ttm_lru_bulk_move_pos *
  79 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
  80 {
  81         return &bulk->pos[res->mem_type][res->bo->priority];
  82 }
  83
  84 /* Move the resource to the tail of the bulk move range */
  85 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
  86                                        struct ttm_resource *res)
  87 {
  88         if (pos->last != res) {
  89                 if (pos->first == res)
  90                         pos->first = list_next_entry(res, lru);
  91                 list_move(&res->lru, &pos->last->lru);
  92                 pos->last = res;
  93         }
  94 }
  95
  96 /* Add the resource to a bulk_move cursor */
  97 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
  98                                   struct ttm_resource *res)
  99 {
 100         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
 101
 102         if (!pos->first) {
 103                 pos->first = res;
 104                 pos->last = res;
 105         } else {
 106                 ttm_lru_bulk_move_pos_tail(pos, res);
 107         }
 108 }
 109
 110 /* Remove the resource from a bulk_move range */
 111 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
 112                                   struct ttm_resource *res)
 113 {
 114         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
 115
 116         if (unlikely(WARN_ON(!pos->first || !pos->last) ||
 117                      (pos->first == res && pos->last == res))) {
 118                 pos->first = NULL;
 119                 pos->last = NULL;
 120         } else if (pos->first == res) {
 121                 pos->first = list_next_entry(res, lru);
 122         } else if (pos->last == res) {
 123                 pos->last = list_prev_entry(res, lru);
 124         } else {
 125                 list_move(&res->lru, &pos->last->lru);
 126         }
 127 }
 128
 129 /* Add the resource to a bulk move if the BO is configured for it */
 130 void ttm_resource_add_bulk_move(struct ttm_resource *res,
 131                                 struct ttm_buffer_object *bo)
 132 {
 133         if (bo->bulk_move && !bo->pin_count)
 134                 ttm_lru_bulk_move_add(bo->bulk_move, res);
 135 }
 136
 137 /* Remove the resource from a bulk move if the BO is configured for it */
 138 void ttm_resource_del_bulk_move(struct ttm_resource *res,
 139                                 struct ttm_buffer_object *bo)
 140 {
 141         if (bo->bulk_move && !bo->pin_count)
 142                 ttm_lru_bulk_move_del(bo->bulk_move, res);
 143 }
 144
 145 /* Move a resource to the LRU or bulk tail */
 146 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
 147 {
 148         struct ttm_buffer_object *bo = res->bo;
 149         struct ttm_device *bdev = bo->bdev;
 150
 151         lockdep_assert_held(&bo->bdev->lru_lock);
 152
 153         if (bo->pin_count) {
 154                 list_move_tail(&res->lru, &bdev->pinned);
 155
 156         } else  if (bo->bulk_move) {
 157                 struct ttm_lru_bulk_move_pos *pos =
 158                         ttm_lru_bulk_move_pos(bo->bulk_move, res);
 159
 160                 ttm_lru_bulk_move_pos_tail(pos, res);
 161         } else {
 162                 struct ttm_resource_manager *man;
 163
 164                 man = ttm_manager_type(bdev, res->mem_type);
 165                 list_move_tail(&res->lru, &man->lru[bo->priority]);
 166         }
 167 }
 168
 169 /**
 170  * ttm_resource_init - resource object constructure
 171  * @bo: buffer object this resources is allocated for
 172  * @place: placement of the resource
 173  * @res: the resource object to inistilize
 174  *
 175  * Initialize a new resource object. Counterpart of ttm_resource_fini().
 176  */
 177 void ttm_resource_init(struct ttm_buffer_object *bo,
 178                        const struct ttm_place *place,
 179                        struct ttm_resource *res)
 180 {
 181         struct ttm_resource_manager *man;
 182
 183         res->start = 0;
 184         res->size = bo->base.size;
 185         res->mem_type = place->mem_type;
 186         res->placement = place->flags;
 187         res->bus.addr = NULL;
 188         res->bus.offset = 0;
 189         res->bus.is_iomem = false;
 190         res->bus.caching = ttm_cached;
 191         res->bo = bo;
 192
 193         man = ttm_manager_type(bo->bdev, place->mem_type);
 194         spin_lock(&bo->bdev->lru_lock);
 195         if (bo->pin_count)
 196                 list_add_tail(&res->lru, &bo->bdev->pinned);
 197         else
 198                 list_add_tail(&res->lru, &man->lru[bo->priority]);
 199         man->usage += res->size;
 200         spin_unlock(&bo->bdev->lru_lock);
 201 }
 202 EXPORT_SYMBOL(ttm_resource_init);
 203
 204 /**
 205  * ttm_resource_fini - resource destructor
 206  * @man: the resource manager this resource belongs to
 207  * @res: the resource to clean up
 208  *
 209  * Should be used by resource manager backends to clean up the TTM resource
 210  * objects before freeing the underlying structure. Makes sure the resource is
 211  * removed from the LRU before destruction.
 212  * Counterpart of ttm_resource_init().
 213  */
 214 void ttm_resource_fini(struct ttm_resource_manager *man,
 215                        struct ttm_resource *res)
 216 {
 217         struct ttm_device *bdev = man->bdev;
 218
 219         spin_lock(&bdev->lru_lock);
 220         list_del_init(&res->lru);
 221         man->usage -= res->size;
 222         spin_unlock(&bdev->lru_lock);
 223 }
 224 EXPORT_SYMBOL(ttm_resource_fini);
 225
 226 int ttm_resource_alloc(struct ttm_buffer_object *bo,
 227                        const struct ttm_place *place,
 228                        struct ttm_resource **res_ptr)
 229 {
 230         struct ttm_resource_manager *man =
 231                 ttm_manager_type(bo->bdev, place->mem_type);
 232         int ret;
 233
 234         ret = man->func->alloc(man, bo, place, res_ptr);
 235         if (ret)
 236                 return ret;
 237
 238         spin_lock(&bo->bdev->lru_lock);
 239         ttm_resource_add_bulk_move(*res_ptr, bo);
 240         spin_unlock(&bo->bdev->lru_lock);
 241         return 0;
 242 }
 243
 244 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
 245 {
 246         struct ttm_resource_manager *man;
 247
 248         if (!*res)
 249                 return;
 250
 251         spin_lock(&bo->bdev->lru_lock);
 252         ttm_resource_del_bulk_move(*res, bo);
 253         spin_unlock(&bo->bdev->lru_lock);
 254         man = ttm_manager_type(bo->bdev, (*res)->mem_type);
 255         man->func->free(man, *res);
 256         *res = NULL;
 257 }
 258 EXPORT_SYMBOL(ttm_resource_free);
 259
 260 /**
 261  * ttm_resource_intersects - test for intersection
 262  *
 263  * @bdev: TTM device structure
 264  * @res: The resource to test
 265  * @place: The placement to test
 266  * @size: How many bytes the new allocation needs.
 267  *
 268  * Test if @res intersects with @place and @size. Used for testing if evictions
 269  * are valueable or not.
 270  *
 271  * Returns true if the res placement intersects with @place and @size.
 272  */
 273 bool ttm_resource_intersects(struct ttm_device *bdev,
 274                              struct ttm_resource *res,
 275                              const struct ttm_place *place,
 276                              size_t size)
 277 {
 278         struct ttm_resource_manager *man;
 279
 280         if (!res)
 281                 return false;
 282
 283         man = ttm_manager_type(bdev, res->mem_type);
 284         if (!place || !man->func->intersects)
 285                 return true;
 286
 287         return man->func->intersects(man, res, place, size);
 288 }
 289
 290 /**
 291  * ttm_resource_compatible - test for compatibility
 292  *
 293  * @bdev: TTM device structure
 294  * @res: The resource to test
 295  * @place: The placement to test
 296  * @size: How many bytes the new allocation needs.
 297  *
 298  * Test if @res compatible with @place and @size.
 299  *
 300  * Returns true if the res placement compatible with @place and @size.
 301  */
 302 bool ttm_resource_compatible(struct ttm_device *bdev,
 303                              struct ttm_resource *res,
 304                              const struct ttm_place *place,
 305                              size_t size)
 306 {
 307         struct ttm_resource_manager *man;
 308
 309         if (!res || !place)
 310                 return false;
 311
 312         man = ttm_manager_type(bdev, res->mem_type);
 313         if (!man->func->compatible)
 314                 return true;
 315
 316         return man->func->compatible(man, res, place, size);
 317 }
 318
 319 static bool ttm_resource_places_compat(struct ttm_resource *res,
 320                                        const struct ttm_place *places,
 321                                        unsigned num_placement)
 322 {
 323         struct ttm_buffer_object *bo = res->bo;
 324         struct ttm_device *bdev = bo->bdev;
 325         unsigned i;
 326
 327         if (res->placement & TTM_PL_FLAG_TEMPORARY)
 328                 return false;
 329
 330         for (i = 0; i < num_placement; i++) {
 331                 const struct ttm_place *heap = &places[i];
 332
 333                 if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
 334                         continue;
 335
 336                 if ((res->mem_type == heap->mem_type) &&
 337                     (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
 338                      (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
 339                         return true;
 340         }
 341         return false;
 342 }
 343
 344 /**
 345  * ttm_resource_compat - check if resource is compatible with placement
 346  *
 347  * @res: the resource to check
 348  * @placement: the placement to check against
 349  *
 350  * Returns true if the placement is compatible.
 351  */
 352 bool ttm_resource_compat(struct ttm_resource *res,
 353                          struct ttm_placement *placement)
 354 {
 355         if (ttm_resource_places_compat(res, placement->placement,
 356                                        placement->num_placement))
 357                 return true;
 358
 359         if ((placement->busy_placement != placement->placement ||
 360              placement->num_busy_placement > placement->num_placement) &&
 361             ttm_resource_places_compat(res, placement->busy_placement,
 362                                        placement->num_busy_placement))
 363                 return true;
 364
 365         return false;
 366 }
 367
 368 void ttm_resource_set_bo(struct ttm_resource *res,
 369                          struct ttm_buffer_object *bo)
 370 {
 371         spin_lock(&bo->bdev->lru_lock);
 372         res->bo = bo;
 373         spin_unlock(&bo->bdev->lru_lock);
 374 }
 375
 376 /**
 377  * ttm_resource_manager_init
 378  *
 379  * @man: memory manager object to init
 380  * @bdev: ttm device this manager belongs to
 381  * @size: size of managed resources in arbitrary units
 382  *
 383  * Initialise core parts of a manager object.
 384  */
 385 void ttm_resource_manager_init(struct ttm_resource_manager *man,
 386                                struct ttm_device *bdev,
 387                                uint64_t size)
 388 {
 389         unsigned i;
 390
 391         spin_lock_init(&man->move_lock);
 392         man->bdev = bdev;
 393         man->size = size;
 394         man->usage = 0;
 395
 396         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
 397                 INIT_LIST_HEAD(&man->lru[i]);
 398         man->move = NULL;
 399 }
 400 EXPORT_SYMBOL(ttm_resource_manager_init);
 401
 402 /*
 403  * ttm_resource_manager_evict_all
 404  *
 405  * @bdev - device to use
 406  * @man - manager to use
 407  *
 408  * Evict all the objects out of a memory manager until it is empty.
 409  * Part of memory manager cleanup sequence.
 410  */
 411 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 412                                    struct ttm_resource_manager *man)
 413 {
 414         struct ttm_operation_ctx ctx = {
 415                 .interruptible = false,
 416                 .no_wait_gpu = false,
 417                 .force_alloc = true
 418         };
 419         struct dma_fence *fence;
 420         int ret;
 421         unsigned i;
 422
 423         /*
 424          * Can't use standard list traversal since we're unlocking.
 425          */
 426
 427         spin_lock(&bdev->lru_lock);
 428         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 429                 while (!list_empty(&man->lru[i])) {
 430                         spin_unlock(&bdev->lru_lock);
 431                         ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
 432                                                   NULL);
 433                         if (ret)
 434                                 return ret;
 435                         spin_lock(&bdev->lru_lock);
 436                 }
 437         }
 438         spin_unlock(&bdev->lru_lock);
 439
 440         spin_lock(&man->move_lock);
 441         fence = dma_fence_get(man->move);
 442         spin_unlock(&man->move_lock);
 443
 444         if (fence) {
 445                 ret = dma_fence_wait(fence, false);
 446                 dma_fence_put(fence);
 447                 if (ret)
 448                         return ret;
 449         }
 450
 451         return 0;
 452 }
 453 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
 454
 455 /**
 456  * ttm_resource_manager_usage
 457  *
 458  * @man: A memory manager object.
 459  *
 460  * Return how many resources are currently used.
 461  */
 462 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
 463 {
 464         uint64_t usage;
 465
 466         spin_lock(&man->bdev->lru_lock);
 467         usage = man->usage;
 468         spin_unlock(&man->bdev->lru_lock);
 469         return usage;
 470 }
 471 EXPORT_SYMBOL(ttm_resource_manager_usage);
 472
 473 /**
 474  * ttm_resource_manager_debug
 475  *
 476  * @man: manager type to dump.
 477  * @p: printer to use for debug.
 478  */
 479 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
 480                                 struct drm_printer *p)
 481 {
 482         drm_printf(p, "  use_type: %d\n", man->use_type);
 483         drm_printf(p, "  use_tt: %d\n", man->use_tt);
 484         drm_printf(p, "  size: %llu\n", man->size);
 485         drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
 486         if (man->func->debug)
 487                 man->func->debug(man, p);
 488 }
 489 EXPORT_SYMBOL(ttm_resource_manager_debug);
 490
 491 /**
 492  * ttm_resource_manager_first
 493  *
 494  * @man: resource manager to iterate over
 495  * @cursor: cursor to record the position
 496  *
 497  * Returns the first resource from the resource manager.
 498  */
 499 struct ttm_resource *
 500 ttm_resource_manager_first(struct ttm_resource_manager *man,
 501                            struct ttm_resource_cursor *cursor)
 502 {
 503         struct ttm_resource *res;
 504
 505         lockdep_assert_held(&man->bdev->lru_lock);
 506
 507         for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
 508              ++cursor->priority)
 509                 list_for_each_entry(res, &man->lru[cursor->priority], lru)
 510                         return res;
 511
 512         return NULL;
 513 }
 514
 515 /**
 516  * ttm_resource_manager_next
 517  *
 518  * @man: resource manager to iterate over
 519  * @cursor: cursor to record the position
 520  * @res: the current resource pointer
 521  *
 522  * Returns the next resource from the resource manager.
 523  */
 524 struct ttm_resource *
 525 ttm_resource_manager_next(struct ttm_resource_manager *man,
 526                           struct ttm_resource_cursor *cursor,
 527                           struct ttm_resource *res)
 528 {
 529         lockdep_assert_held(&man->bdev->lru_lock);
 530
 531         list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
 532                 return res;
 533
 534         for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
 535              ++cursor->priority)
 536                 list_for_each_entry(res, &man->lru[cursor->priority], lru)
 537                         return res;
 538
 539         return NULL;
 540 }
 541
 542 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
 543                                           struct iosys_map *dmap,
 544                                           pgoff_t i)
 545 {
 546         struct ttm_kmap_iter_iomap *iter_io =
 547                 container_of(iter, typeof(*iter_io), base);
 548         void __iomem *addr;
 549
 550 retry:
 551         while (i >= iter_io->cache.end) {
 552                 iter_io->cache.sg = iter_io->cache.sg ?
 553                         sg_next(iter_io->cache.sg) : iter_io->st->sgl;
 554                 iter_io->cache.i = iter_io->cache.end;
 555                 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
 556                         PAGE_SHIFT;
 557                 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
 558                         iter_io->start;
 559         }
 560
 561         if (i < iter_io->cache.i) {
 562                 iter_io->cache.end = 0;
 563                 iter_io->cache.sg = NULL;
 564                 goto retry;
 565         }
 566
 567         addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
 568                                        (((resource_size_t)i - iter_io->cache.i)
 569                                         << PAGE_SHIFT));
 570         iosys_map_set_vaddr_iomem(dmap, addr);
 571 }
 572
 573 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
 574                                             struct iosys_map *map)
 575 {
 576         io_mapping_unmap_local(map->vaddr_iomem);
 577 }
 578
 579 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
 580         .map_local =  ttm_kmap_iter_iomap_map_local,
 581         .unmap_local = ttm_kmap_iter_iomap_unmap_local,
 582         .maps_tt = false,
 583 };
 584
 585 /**
 586  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
 587  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
 588  * @iomap: The struct io_mapping representing the underlying linear io_memory.
 589  * @st: sg_table into @iomap, representing the memory of the struct
 590  * ttm_resource.
 591  * @start: Offset that needs to be subtracted from @st to make
 592  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
 593  *
 594  * Return: Pointer to the embedded struct ttm_kmap_iter.
 595  */
 596 struct ttm_kmap_iter *
 597 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
 598                          struct io_mapping *iomap,
 599                          struct sg_table *st,
 600                          resource_size_t start)
 601 {
 602         iter_io->base.ops = &ttm_kmap_iter_io_ops;
 603         iter_io->iomap = iomap;
 604         iter_io->st = st;
 605         iter_io->start = start;
 606         memset(&iter_io->cache, 0, sizeof(iter_io->cache));
 607
 608         return &iter_io->base;
 609 }
 610 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
 611
 612 /**
 613  * DOC: Linear io iterator
 614  *
 615  * This code should die in the not too near future. Best would be if we could
 616  * make io-mapping use memremap for all io memory, and have memremap
 617  * implement a kmap_local functionality. We could then strip a huge amount of
 618  * code. These linear io iterators are implemented to mimic old functionality,
 619  * and they don't use kmap_local semantics at all internally. Rather ioremap or
 620  * friends, and at least on 32-bit they add global TLB flushes and points
 621  * of failure.
 622  */
 623
 624 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
 625                                               struct iosys_map *dmap,
 626                                               pgoff_t i)
 627 {
 628         struct ttm_kmap_iter_linear_io *iter_io =
 629                 container_of(iter, typeof(*iter_io), base);
 630
 631         *dmap = iter_io->dmap;
 632         iosys_map_incr(dmap, i * PAGE_SIZE);
 633 }
 634
 635 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
 636         .map_local =  ttm_kmap_iter_linear_io_map_local,
 637         .maps_tt = false,
 638 };
 639
 640 /**
 641  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
 642  * @iter_io: The iterator to initialize
 643  * @bdev: The TTM device
 644  * @mem: The ttm resource representing the iomap.
 645  *
 646  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
 647  * pointing at a linear chunk of io memory.
 648  *
 649  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
 650  * failure.
 651  */
 652 struct ttm_kmap_iter *
 653 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 654                              struct ttm_device *bdev,
 655                              struct ttm_resource *mem)
 656 {
 657         int ret;
 658
 659         ret = ttm_mem_io_reserve(bdev, mem);
 660         if (ret)
 661                 goto out_err;
 662         if (!mem->bus.is_iomem) {
 663                 ret = -EINVAL;
 664                 goto out_io_free;
 665         }
 666
 667         if (mem->bus.addr) {
 668                 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
 669                 iter_io->needs_unmap = false;
 670         } else {
 671                 iter_io->needs_unmap = true;
 672                 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
 673                 if (mem->bus.caching == ttm_write_combined)
 674                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
 675                                                   ioremap_wc(mem->bus.offset,
 676                                                              mem->size));
 677                 else if (mem->bus.caching == ttm_cached)
 678                         iosys_map_set_vaddr(&iter_io->dmap,
 679                                             memremap(mem->bus.offset, mem->size,
 680                                                      MEMREMAP_WB |
 681                                                      MEMREMAP_WT |
 682                                                      MEMREMAP_WC));
 683
 684                 /* If uncached requested or if mapping cached or wc failed */
 685                 if (iosys_map_is_null(&iter_io->dmap))
 686                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
 687                                                   ioremap(mem->bus.offset,
 688                                                           mem->size));
 689
 690                 if (iosys_map_is_null(&iter_io->dmap)) {
 691                         ret = -ENOMEM;
 692                         goto out_io_free;
 693                 }
 694         }
 695
 696         iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
 697         return &iter_io->base;
 698
 699 out_io_free:
 700         ttm_mem_io_free(bdev, mem);
 701 out_err:
 702         return ERR_PTR(ret);
 703 }
 704
 705 /**
 706  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
 707  * @iter_io: The iterator to initialize
 708  * @bdev: The TTM device
 709  * @mem: The ttm resource representing the iomap.
 710  *
 711  * This function is for internal TTM use only. It cleans up a memcpy kmap
 712  * iterator initialized by ttm_kmap_iter_linear_io_init.
 713  */
 714 void
 715 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
 716                              struct ttm_device *bdev,
 717                              struct ttm_resource *mem)
 718 {
 719         if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
 720                 if (iter_io->dmap.is_iomem)
 721                         iounmap(iter_io->dmap.vaddr_iomem);
 722                 else
 723                         memunmap(iter_io->dmap.vaddr);
 724         }
 725
 726         ttm_mem_io_free(bdev, mem);
 727 }
 728
 729 #if defined(CONFIG_DEBUG_FS)
 730
 731 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
 732 {
 733         struct ttm_resource_manager *man =
 734                 (struct ttm_resource_manager *)m->private;
 735         struct drm_printer p = drm_seq_file_printer(m);
 736         ttm_resource_manager_debug(man, &p);
 737         return 0;
 738 }
 739 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
 740
 741 #endif
 742
 743 /**
 744  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
 745  * resource manager.
 746  * @man: The TTM resource manager for which the debugfs stats file be creates
 747  * @parent: debugfs directory in which the file will reside
 748  * @name: The filename to create.
 749  *
 750  * This function setups up a debugfs file that can be used to look
 751  * at debug statistics of the specified ttm_resource_manager.
 752  */
 753 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
 754                                          struct dentry * parent,
 755                                          const char *name)
 756 {
 757 #if defined(CONFIG_DEBUG_FS)
 758         debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
 759 #endif
 760 }
 761 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);