drivers/gpu/drm/ttm/ttm_resource.c

   1 /*
   2  * Copyright 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Christian König
  23  */
  24
  25 #include <linux/iosys-map.h>
  26 #include <linux/io-mapping.h>
  27 #include <linux/scatterlist.h>
  28
  29 #include <drm/ttm/ttm_bo.h>
  30 #include <drm/ttm/ttm_placement.h>
  31 #include <drm/ttm/ttm_resource.h>
  32
  33 /**
  34  * ttm_lru_bulk_move_init - initialize a bulk move structure
  35  * @bulk: the structure to init
  36  *
  37  * For now just memset the structure to zero.
  38  */
  39 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
  40 {
  41         memset(bulk, 0, sizeof(*bulk));
  42 }
  43 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
  44
  45 /**
  46  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
  47  *
  48  * @bulk: bulk move structure
  49  *
  50  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
  51  * resource order never changes. Should be called with &ttm_device.lru_lock held.
  52  */
  53 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
  54 {
  55         unsigned i, j;
  56
  57         for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
  58                 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
  59                         struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
  60                         struct ttm_resource_manager *man;
  61
  62                         if (!pos->first)
  63                                 continue;
  64
  65                         lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
  66                         dma_resv_assert_held(pos->first->bo->base.resv);
  67                         dma_resv_assert_held(pos->last->bo->base.resv);
  68
  69                         man = ttm_manager_type(pos->first->bo->bdev, i);
  70                         list_bulk_move_tail(&man->lru[j], &pos->first->lru,
  71                                             &pos->last->lru);
  72                 }
  73         }
  74 }
  75 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
  76
  77 /* Return the bulk move pos object for this resource */
  78 static struct ttm_lru_bulk_move_pos *
  79 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
  80 {
  81         return &bulk->pos[res->mem_type][res->bo->priority];
  82 }
  83
  84 /* Move the resource to the tail of the bulk move range */
  85 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
  86                                        struct ttm_resource *res)
  87 {
  88         if (pos->last != res) {
  89                 list_move(&res->lru, &pos->last->lru);
  90                 pos->last = res;
  91         }
  92 }
  93
  94 /* Add the resource to a bulk_move cursor */
  95 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
  96                                   struct ttm_resource *res)
  97 {
  98         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
  99
 100         if (!pos->first) {
 101                 pos->first = res;
 102                 pos->last = res;
 103         } else {
 104                 ttm_lru_bulk_move_pos_tail(pos, res);
 105         }
 106 }
 107
 108 /* Remove the resource from a bulk_move range */
 109 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
 110                                   struct ttm_resource *res)
 111 {
 112         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
 113
 114         if (unlikely(pos->first == res && pos->last == res)) {
 115                 pos->first = NULL;
 116                 pos->last = NULL;
 117         } else if (pos->first == res) {
 118                 pos->first = list_next_entry(res, lru);
 119         } else if (pos->last == res) {
 120                 pos->last = list_prev_entry(res, lru);
 121         } else {
 122                 list_move(&res->lru, &pos->last->lru);
 123         }
 124 }
 125
 126 /* Add the resource to a bulk move if the BO is configured for it */
 127 void ttm_resource_add_bulk_move(struct ttm_resource *res,
 128                                 struct ttm_buffer_object *bo)
 129 {
 130         if (bo->bulk_move && !bo->pin_count)
 131                 ttm_lru_bulk_move_add(bo->bulk_move, res);
 132 }
 133
 134 /* Remove the resource from a bulk move if the BO is configured for it */
 135 void ttm_resource_del_bulk_move(struct ttm_resource *res,
 136                                 struct ttm_buffer_object *bo)
 137 {
 138         if (bo->bulk_move && !bo->pin_count)
 139                 ttm_lru_bulk_move_del(bo->bulk_move, res);
 140 }
 141
 142 /* Move a resource to the LRU or bulk tail */
 143 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
 144 {
 145         struct ttm_buffer_object *bo = res->bo;
 146         struct ttm_device *bdev = bo->bdev;
 147
 148         lockdep_assert_held(&bo->bdev->lru_lock);
 149
 150         if (bo->pin_count) {
 151                 list_move_tail(&res->lru, &bdev->pinned);
 152
 153         } else  if (bo->bulk_move) {
 154                 struct ttm_lru_bulk_move_pos *pos =
 155                         ttm_lru_bulk_move_pos(bo->bulk_move, res);
 156
 157                 ttm_lru_bulk_move_pos_tail(pos, res);
 158         } else {
 159                 struct ttm_resource_manager *man;
 160
 161                 man = ttm_manager_type(bdev, res->mem_type);
 162                 list_move_tail(&res->lru, &man->lru[bo->priority]);
 163         }
 164 }
 165
 166 /**
 167  * ttm_resource_init - resource object constructure
 168  * @bo: buffer object this resources is allocated for
 169  * @place: placement of the resource
 170  * @res: the resource object to inistilize
 171  *
 172  * Initialize a new resource object. Counterpart of ttm_resource_fini().
 173  */
 174 void ttm_resource_init(struct ttm_buffer_object *bo,
 175                        const struct ttm_place *place,
 176                        struct ttm_resource *res)
 177 {
 178         struct ttm_resource_manager *man;
 179
 180         res->start = 0;
 181         res->size = bo->base.size;
 182         res->mem_type = place->mem_type;
 183         res->placement = place->flags;
 184         res->bus.addr = NULL;
 185         res->bus.offset = 0;
 186         res->bus.is_iomem = false;
 187         res->bus.caching = ttm_cached;
 188         res->bo = bo;
 189
 190         man = ttm_manager_type(bo->bdev, place->mem_type);
 191         spin_lock(&bo->bdev->lru_lock);
 192         if (bo->pin_count)
 193                 list_add_tail(&res->lru, &bo->bdev->pinned);
 194         else
 195                 list_add_tail(&res->lru, &man->lru[bo->priority]);
 196         man->usage += res->size;
 197         spin_unlock(&bo->bdev->lru_lock);
 198 }
 199 EXPORT_SYMBOL(ttm_resource_init);
 200
 201 /**
 202  * ttm_resource_fini - resource destructor
 203  * @man: the resource manager this resource belongs to
 204  * @res: the resource to clean up
 205  *
 206  * Should be used by resource manager backends to clean up the TTM resource
 207  * objects before freeing the underlying structure. Makes sure the resource is
 208  * removed from the LRU before destruction.
 209  * Counterpart of ttm_resource_init().
 210  */
 211 void ttm_resource_fini(struct ttm_resource_manager *man,
 212                        struct ttm_resource *res)
 213 {
 214         struct ttm_device *bdev = man->bdev;
 215
 216         spin_lock(&bdev->lru_lock);
 217         list_del_init(&res->lru);
 218         man->usage -= res->size;
 219         spin_unlock(&bdev->lru_lock);
 220 }
 221 EXPORT_SYMBOL(ttm_resource_fini);
 222
 223 int ttm_resource_alloc(struct ttm_buffer_object *bo,
 224                        const struct ttm_place *place,
 225                        struct ttm_resource **res_ptr)
 226 {
 227         struct ttm_resource_manager *man =
 228                 ttm_manager_type(bo->bdev, place->mem_type);
 229         int ret;
 230
 231         ret = man->func->alloc(man, bo, place, res_ptr);
 232         if (ret)
 233                 return ret;
 234
 235         spin_lock(&bo->bdev->lru_lock);
 236         ttm_resource_add_bulk_move(*res_ptr, bo);
 237         spin_unlock(&bo->bdev->lru_lock);
 238         return 0;
 239 }
 240
 241 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
 242 {
 243         struct ttm_resource_manager *man;
 244
 245         if (!*res)
 246                 return;
 247
 248         spin_lock(&bo->bdev->lru_lock);
 249         ttm_resource_del_bulk_move(*res, bo);
 250         spin_unlock(&bo->bdev->lru_lock);
 251         man = ttm_manager_type(bo->bdev, (*res)->mem_type);
 252         man->func->free(man, *res);
 253         *res = NULL;
 254 }
 255 EXPORT_SYMBOL(ttm_resource_free);
 256
 257 /**
 258  * ttm_resource_intersects - test for intersection
 259  *
 260  * @bdev: TTM device structure
 261  * @res: The resource to test
 262  * @place: The placement to test
 263  * @size: How many bytes the new allocation needs.
 264  *
 265  * Test if @res intersects with @place and @size. Used for testing if evictions
 266  * are valueable or not.
 267  *
 268  * Returns true if the res placement intersects with @place and @size.
 269  */
 270 bool ttm_resource_intersects(struct ttm_device *bdev,
 271                              struct ttm_resource *res,
 272                              const struct ttm_place *place,
 273                              size_t size)
 274 {
 275         struct ttm_resource_manager *man;
 276
 277         if (!res)
 278                 return false;
 279
 280         man = ttm_manager_type(bdev, res->mem_type);
 281         if (!place || !man->func->intersects)
 282                 return true;
 283
 284         return man->func->intersects(man, res, place, size);
 285 }
 286
 287 /**
 288  * ttm_resource_compatible - test for compatibility
 289  *
 290  * @bdev: TTM device structure
 291  * @res: The resource to test
 292  * @place: The placement to test
 293  * @size: How many bytes the new allocation needs.
 294  *
 295  * Test if @res compatible with @place and @size.
 296  *
 297  * Returns true if the res placement compatible with @place and @size.
 298  */
 299 bool ttm_resource_compatible(struct ttm_device *bdev,
 300                              struct ttm_resource *res,
 301                              const struct ttm_place *place,
 302                              size_t size)
 303 {
 304         struct ttm_resource_manager *man;
 305
 306         if (!res || !place)
 307                 return false;
 308
 309         man = ttm_manager_type(bdev, res->mem_type);
 310         if (!man->func->compatible)
 311                 return true;
 312
 313         return man->func->compatible(man, res, place, size);
 314 }
 315
 316 static bool ttm_resource_places_compat(struct ttm_resource *res,
 317                                        const struct ttm_place *places,
 318                                        unsigned num_placement)
 319 {
 320         struct ttm_buffer_object *bo = res->bo;
 321         struct ttm_device *bdev = bo->bdev;
 322         unsigned i;
 323
 324         if (res->placement & TTM_PL_FLAG_TEMPORARY)
 325                 return false;
 326
 327         for (i = 0; i < num_placement; i++) {
 328                 const struct ttm_place *heap = &places[i];
 329
 330                 if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
 331                         continue;
 332
 333                 if ((res->mem_type == heap->mem_type) &&
 334                     (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
 335                      (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
 336                         return true;
 337         }
 338         return false;
 339 }
 340
 341 /**
 342  * ttm_resource_compat - check if resource is compatible with placement
 343  *
 344  * @res: the resource to check
 345  * @placement: the placement to check against
 346  *
 347  * Returns true if the placement is compatible.
 348  */
 349 bool ttm_resource_compat(struct ttm_resource *res,
 350                          struct ttm_placement *placement)
 351 {
 352         if (ttm_resource_places_compat(res, placement->placement,
 353                                        placement->num_placement))
 354                 return true;
 355
 356         if ((placement->busy_placement != placement->placement ||
 357              placement->num_busy_placement > placement->num_placement) &&
 358             ttm_resource_places_compat(res, placement->busy_placement,
 359                                        placement->num_busy_placement))
 360                 return true;
 361
 362         return false;
 363 }
 364
 365 void ttm_resource_set_bo(struct ttm_resource *res,
 366                          struct ttm_buffer_object *bo)
 367 {
 368         spin_lock(&bo->bdev->lru_lock);
 369         res->bo = bo;
 370         spin_unlock(&bo->bdev->lru_lock);
 371 }
 372
 373 /**
 374  * ttm_resource_manager_init
 375  *
 376  * @man: memory manager object to init
 377  * @bdev: ttm device this manager belongs to
 378  * @size: size of managed resources in arbitrary units
 379  *
 380  * Initialise core parts of a manager object.
 381  */
 382 void ttm_resource_manager_init(struct ttm_resource_manager *man,
 383                                struct ttm_device *bdev,
 384                                uint64_t size)
 385 {
 386         unsigned i;
 387
 388         spin_lock_init(&man->move_lock);
 389         man->bdev = bdev;
 390         man->size = size;
 391         man->usage = 0;
 392
 393         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
 394                 INIT_LIST_HEAD(&man->lru[i]);
 395         man->move = NULL;
 396 }
 397 EXPORT_SYMBOL(ttm_resource_manager_init);
 398
 399 /*
 400  * ttm_resource_manager_evict_all
 401  *
 402  * @bdev - device to use
 403  * @man - manager to use
 404  *
 405  * Evict all the objects out of a memory manager until it is empty.
 406  * Part of memory manager cleanup sequence.
 407  */
 408 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 409                                    struct ttm_resource_manager *man)
 410 {
 411         struct ttm_operation_ctx ctx = {
 412                 .interruptible = false,
 413                 .no_wait_gpu = false,
 414                 .force_alloc = true
 415         };
 416         struct dma_fence *fence;
 417         int ret;
 418         unsigned i;
 419
 420         /*
 421          * Can't use standard list traversal since we're unlocking.
 422          */
 423
 424         spin_lock(&bdev->lru_lock);
 425         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 426                 while (!list_empty(&man->lru[i])) {
 427                         spin_unlock(&bdev->lru_lock);
 428                         ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
 429                                                   NULL);
 430                         if (ret)
 431                                 return ret;
 432                         spin_lock(&bdev->lru_lock);
 433                 }
 434         }
 435         spin_unlock(&bdev->lru_lock);
 436
 437         spin_lock(&man->move_lock);
 438         fence = dma_fence_get(man->move);
 439         spin_unlock(&man->move_lock);
 440
 441         if (fence) {
 442                 ret = dma_fence_wait(fence, false);
 443                 dma_fence_put(fence);
 444                 if (ret)
 445                         return ret;
 446         }
 447
 448         return 0;
 449 }
 450 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
 451
 452 /**
 453  * ttm_resource_manager_usage
 454  *
 455  * @man: A memory manager object.
 456  *
 457  * Return how many resources are currently used.
 458  */
 459 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
 460 {
 461         uint64_t usage;
 462
 463         spin_lock(&man->bdev->lru_lock);
 464         usage = man->usage;
 465         spin_unlock(&man->bdev->lru_lock);
 466         return usage;
 467 }
 468 EXPORT_SYMBOL(ttm_resource_manager_usage);
 469
 470 /**
 471  * ttm_resource_manager_debug
 472  *
 473  * @man: manager type to dump.
 474  * @p: printer to use for debug.
 475  */
 476 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
 477                                 struct drm_printer *p)
 478 {
 479         drm_printf(p, "  use_type: %d\n", man->use_type);
 480         drm_printf(p, "  use_tt: %d\n", man->use_tt);
 481         drm_printf(p, "  size: %llu\n", man->size);
 482         drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
 483         if (man->func->debug)
 484                 man->func->debug(man, p);
 485 }
 486 EXPORT_SYMBOL(ttm_resource_manager_debug);
 487
 488 /**
 489  * ttm_resource_manager_first
 490  *
 491  * @man: resource manager to iterate over
 492  * @cursor: cursor to record the position
 493  *
 494  * Returns the first resource from the resource manager.
 495  */
 496 struct ttm_resource *
 497 ttm_resource_manager_first(struct ttm_resource_manager *man,
 498                            struct ttm_resource_cursor *cursor)
 499 {
 500         struct ttm_resource *res;
 501
 502         lockdep_assert_held(&man->bdev->lru_lock);
 503
 504         for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
 505              ++cursor->priority)
 506                 list_for_each_entry(res, &man->lru[cursor->priority], lru)
 507                         return res;
 508
 509         return NULL;
 510 }
 511
 512 /**
 513  * ttm_resource_manager_next
 514  *
 515  * @man: resource manager to iterate over
 516  * @cursor: cursor to record the position
 517  * @res: the current resource pointer
 518  *
 519  * Returns the next resource from the resource manager.
 520  */
 521 struct ttm_resource *
 522 ttm_resource_manager_next(struct ttm_resource_manager *man,
 523                           struct ttm_resource_cursor *cursor,
 524                           struct ttm_resource *res)
 525 {
 526         lockdep_assert_held(&man->bdev->lru_lock);
 527
 528         list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
 529                 return res;
 530
 531         for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
 532              ++cursor->priority)
 533                 list_for_each_entry(res, &man->lru[cursor->priority], lru)
 534                         return res;
 535
 536         return NULL;
 537 }
 538
 539 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
 540                                           struct iosys_map *dmap,
 541                                           pgoff_t i)
 542 {
 543         struct ttm_kmap_iter_iomap *iter_io =
 544                 container_of(iter, typeof(*iter_io), base);
 545         void __iomem *addr;
 546
 547 retry:
 548         while (i >= iter_io->cache.end) {
 549                 iter_io->cache.sg = iter_io->cache.sg ?
 550                         sg_next(iter_io->cache.sg) : iter_io->st->sgl;
 551                 iter_io->cache.i = iter_io->cache.end;
 552                 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
 553                         PAGE_SHIFT;
 554                 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
 555                         iter_io->start;
 556         }
 557
 558         if (i < iter_io->cache.i) {
 559                 iter_io->cache.end = 0;
 560                 iter_io->cache.sg = NULL;
 561                 goto retry;
 562         }
 563
 564         addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
 565                                        (((resource_size_t)i - iter_io->cache.i)
 566                                         << PAGE_SHIFT));
 567         iosys_map_set_vaddr_iomem(dmap, addr);
 568 }
 569
 570 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
 571                                             struct iosys_map *map)
 572 {
 573         io_mapping_unmap_local(map->vaddr_iomem);
 574 }
 575
 576 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
 577         .map_local =  ttm_kmap_iter_iomap_map_local,
 578         .unmap_local = ttm_kmap_iter_iomap_unmap_local,
 579         .maps_tt = false,
 580 };
 581
 582 /**
 583  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
 584  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
 585  * @iomap: The struct io_mapping representing the underlying linear io_memory.
 586  * @st: sg_table into @iomap, representing the memory of the struct
 587  * ttm_resource.
 588  * @start: Offset that needs to be subtracted from @st to make
 589  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
 590  *
 591  * Return: Pointer to the embedded struct ttm_kmap_iter.
 592  */
 593 struct ttm_kmap_iter *
 594 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
 595                          struct io_mapping *iomap,
 596                          struct sg_table *st,
 597                          resource_size_t start)
 598 {
 599         iter_io->base.ops = &ttm_kmap_iter_io_ops;
 600         iter_io->iomap = iomap;
 601         iter_io->st = st;
 602         iter_io->start = start;
 603         memset(&iter_io->cache, 0, sizeof(iter_io->cache));
 604
 605         return &iter_io->base;
 606 }
 607 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
 608
 609 /**
 610  * DOC: Linear io iterator
 611  *
 612  * This code should die in the not too near future. Best would be if we could
 613  * make io-mapping use memremap for all io memory, and have memremap
 614  * implement a kmap_local functionality. We could then strip a huge amount of
 615  * code. These linear io iterators are implemented to mimic old functionality,
 616  * and they don't use kmap_local semantics at all internally. Rather ioremap or
 617  * friends, and at least on 32-bit they add global TLB flushes and points
 618  * of failure.
 619  */
 620
 621 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
 622                                               struct iosys_map *dmap,
 623                                               pgoff_t i)
 624 {
 625         struct ttm_kmap_iter_linear_io *iter_io =
 626                 container_of(iter, typeof(*iter_io), base);
 627
 628         *dmap = iter_io->dmap;
 629         iosys_map_incr(dmap, i * PAGE_SIZE);
 630 }
 631
 632 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
 633         .map_local =  ttm_kmap_iter_linear_io_map_local,
 634         .maps_tt = false,
 635 };
 636
 637 /**
 638  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
 639  * @iter_io: The iterator to initialize
 640  * @bdev: The TTM device
 641  * @mem: The ttm resource representing the iomap.
 642  *
 643  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
 644  * pointing at a linear chunk of io memory.
 645  *
 646  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
 647  * failure.
 648  */
 649 struct ttm_kmap_iter *
 650 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 651                              struct ttm_device *bdev,
 652                              struct ttm_resource *mem)
 653 {
 654         int ret;
 655
 656         ret = ttm_mem_io_reserve(bdev, mem);
 657         if (ret)
 658                 goto out_err;
 659         if (!mem->bus.is_iomem) {
 660                 ret = -EINVAL;
 661                 goto out_io_free;
 662         }
 663
 664         if (mem->bus.addr) {
 665                 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
 666                 iter_io->needs_unmap = false;
 667         } else {
 668                 iter_io->needs_unmap = true;
 669                 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
 670                 if (mem->bus.caching == ttm_write_combined)
 671                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
 672                                                   ioremap_wc(mem->bus.offset,
 673                                                              mem->size));
 674                 else if (mem->bus.caching == ttm_cached)
 675                         iosys_map_set_vaddr(&iter_io->dmap,
 676                                             memremap(mem->bus.offset, mem->size,
 677                                                      MEMREMAP_WB |
 678                                                      MEMREMAP_WT |
 679                                                      MEMREMAP_WC));
 680
 681                 /* If uncached requested or if mapping cached or wc failed */
 682                 if (iosys_map_is_null(&iter_io->dmap))
 683                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
 684                                                   ioremap(mem->bus.offset,
 685                                                           mem->size));
 686
 687                 if (iosys_map_is_null(&iter_io->dmap)) {
 688                         ret = -ENOMEM;
 689                         goto out_io_free;
 690                 }
 691         }
 692
 693         iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
 694         return &iter_io->base;
 695
 696 out_io_free:
 697         ttm_mem_io_free(bdev, mem);
 698 out_err:
 699         return ERR_PTR(ret);
 700 }
 701
 702 /**
 703  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
 704  * @iter_io: The iterator to initialize
 705  * @bdev: The TTM device
 706  * @mem: The ttm resource representing the iomap.
 707  *
 708  * This function is for internal TTM use only. It cleans up a memcpy kmap
 709  * iterator initialized by ttm_kmap_iter_linear_io_init.
 710  */
 711 void
 712 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
 713                              struct ttm_device *bdev,
 714                              struct ttm_resource *mem)
 715 {
 716         if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
 717                 if (iter_io->dmap.is_iomem)
 718                         iounmap(iter_io->dmap.vaddr_iomem);
 719                 else
 720                         memunmap(iter_io->dmap.vaddr);
 721         }
 722
 723         ttm_mem_io_free(bdev, mem);
 724 }
 725
 726 #if defined(CONFIG_DEBUG_FS)
 727
 728 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
 729 {
 730         struct ttm_resource_manager *man =
 731                 (struct ttm_resource_manager *)m->private;
 732         struct drm_printer p = drm_seq_file_printer(m);
 733         ttm_resource_manager_debug(man, &p);
 734         return 0;
 735 }
 736 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
 737
 738 #endif
 739
 740 /**
 741  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
 742  * resource manager.
 743  * @man: The TTM resource manager for which the debugfs stats file be creates
 744  * @parent: debugfs directory in which the file will reside
 745  * @name: The filename to create.
 746  *
 747  * This function setups up a debugfs file that can be used to look
 748  * at debug statistics of the specified ttm_resource_manager.
 749  */
 750 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
 751                                          struct dentry * parent,
 752                                          const char *name)
 753 {
 754 #if defined(CONFIG_DEBUG_FS)
 755         debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
 756 #endif
 757 }
 758 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);