drivers/gpu/arm/midgard/r12p0_04rel0/mali_kbase_mem_pool.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 #include <mali_kbase.h>
  19 #include <linux/mm.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/highmem.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/shrinker.h>
  24 #include <linux/atomic.h>
  25 #include <linux/version.h>
  26
  27 /* This function is only provided for backwards compatibility with kernels
  28  * which use the old carveout allocator.
  29  *
  30  * The forward declaration is to keep sparse happy.
  31  */
  32 int __init kbase_carveout_mem_reserve(
  33                 phys_addr_t size);
  34 int __init kbase_carveout_mem_reserve(phys_addr_t size)
  35 {
  36         return 0;
  37 }
  38
  39 #define pool_dbg(pool, format, ...) \
  40         dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \
  41                 (pool->next_pool) ? "kctx" : "kbdev",   \
  42                 kbase_mem_pool_size(pool),      \
  43                 kbase_mem_pool_max_size(pool),  \
  44                 ##__VA_ARGS__)
  45
  46 #define NOT_DIRTY false
  47 #define NOT_RECLAIMED false
  48
  49 static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool)
  50 {
  51         spin_lock(&pool->pool_lock);
  52 }
  53
  54 static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool)
  55 {
  56         spin_unlock(&pool->pool_lock);
  57 }
  58
  59 static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool)
  60 {
  61         ssize_t max_size = kbase_mem_pool_max_size(pool);
  62         ssize_t cur_size = kbase_mem_pool_size(pool);
  63
  64         return max(max_size - cur_size, (ssize_t)0);
  65 }
  66
  67 static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool)
  68 {
  69         return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool);
  70 }
  71
  72 static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool)
  73 {
  74         return kbase_mem_pool_size(pool) == 0;
  75 }
  76
  77 static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool,
  78                 struct page *p)
  79 {
  80         lockdep_assert_held(&pool->pool_lock);
  81
  82         list_add(&p->lru, &pool->page_list);
  83         pool->cur_size++;
  84
  85         zone_page_state_add(1, page_zone(p), NR_SLAB_RECLAIMABLE);
  86
  87         pool_dbg(pool, "added page\n");
  88 }
  89
  90 static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p)
  91 {
  92         kbase_mem_pool_lock(pool);
  93         kbase_mem_pool_add_locked(pool, p);
  94         kbase_mem_pool_unlock(pool);
  95 }
  96
  97 static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool,
  98                 struct list_head *page_list, size_t nr_pages)
  99 {
 100         struct page *p;
 101
 102         lockdep_assert_held(&pool->pool_lock);
 103
 104         list_for_each_entry(p, page_list, lru) {
 105                 zone_page_state_add(1, page_zone(p), NR_SLAB_RECLAIMABLE);
 106         }
 107
 108         list_splice(page_list, &pool->page_list);
 109         pool->cur_size += nr_pages;
 110
 111         pool_dbg(pool, "added %zu pages\n", nr_pages);
 112 }
 113
 114 static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool,
 115                 struct list_head *page_list, size_t nr_pages)
 116 {
 117         kbase_mem_pool_lock(pool);
 118         kbase_mem_pool_add_list_locked(pool, page_list, nr_pages);
 119         kbase_mem_pool_unlock(pool);
 120 }
 121
 122 static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool)
 123 {
 124         struct page *p;
 125
 126         lockdep_assert_held(&pool->pool_lock);
 127
 128         if (kbase_mem_pool_is_empty(pool))
 129                 return NULL;
 130
 131         p = list_first_entry(&pool->page_list, struct page, lru);
 132         list_del_init(&p->lru);
 133         pool->cur_size--;
 134
 135         zone_page_state_add(-1, page_zone(p), NR_SLAB_RECLAIMABLE);
 136
 137         pool_dbg(pool, "removed page\n");
 138
 139         return p;
 140 }
 141
 142 static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool)
 143 {
 144         struct page *p;
 145
 146         kbase_mem_pool_lock(pool);
 147         p = kbase_mem_pool_remove_locked(pool);
 148         kbase_mem_pool_unlock(pool);
 149
 150         return p;
 151 }
 152
 153 static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool,
 154                 struct page *p)
 155 {
 156         struct device *dev = pool->kbdev->dev;
 157
 158         dma_sync_single_for_device(dev, kbase_dma_addr(p),
 159                         PAGE_SIZE, DMA_BIDIRECTIONAL);
 160 }
 161
 162 static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool,
 163                 struct page *p)
 164 {
 165         clear_highpage(p);
 166         kbase_mem_pool_sync_page(pool, p);
 167 }
 168
 169 static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool,
 170                 struct page *p)
 171 {
 172         /* Zero page before spilling */
 173         kbase_mem_pool_zero_page(next_pool, p);
 174
 175         kbase_mem_pool_add(next_pool, p);
 176 }
 177
 178 static struct page *kbase_mem_pool_alloc_page(struct kbase_mem_pool *pool)
 179 {
 180         struct page *p;
 181         gfp_t gfp;
 182         struct device *dev = pool->kbdev->dev;
 183         dma_addr_t dma_addr;
 184
 185 #if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \
 186         LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
 187         /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
 188         gfp = GFP_USER | __GFP_ZERO;
 189 #else
 190         gfp = GFP_HIGHUSER | __GFP_ZERO;
 191 #endif
 192
 193         if (current->flags & PF_KTHREAD) {
 194                 /* Don't trigger OOM killer from kernel threads, e.g. when
 195                  * growing memory on GPU page fault */
 196                 gfp |= __GFP_NORETRY;
 197         }
 198
 199         p = alloc_page(gfp);
 200         if (!p)
 201                 return NULL;
 202
 203         dma_addr = dma_map_page(dev, p, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
 204         if (dma_mapping_error(dev, dma_addr)) {
 205                 __free_page(p);
 206                 return NULL;
 207         }
 208
 209         WARN_ON(dma_addr != page_to_phys(p));
 210
 211         kbase_set_dma_addr(p, dma_addr);
 212
 213         pool_dbg(pool, "alloced page from kernel\n");
 214
 215         return p;
 216 }
 217
 218 static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool,
 219                 struct page *p)
 220 {
 221         struct device *dev = pool->kbdev->dev;
 222         dma_addr_t dma_addr = kbase_dma_addr(p);
 223
 224         dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 225         kbase_clear_dma_addr(p);
 226         __free_page(p);
 227
 228         pool_dbg(pool, "freed page to kernel\n");
 229 }
 230
 231 static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool,
 232                 size_t nr_to_shrink)
 233 {
 234         struct page *p;
 235         size_t i;
 236
 237         lockdep_assert_held(&pool->pool_lock);
 238
 239         for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) {
 240                 p = kbase_mem_pool_remove_locked(pool);
 241                 kbase_mem_pool_free_page(pool, p);
 242         }
 243
 244         return i;
 245 }
 246
 247 static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool,
 248                 size_t nr_to_shrink)
 249 {
 250         size_t nr_freed;
 251
 252         kbase_mem_pool_lock(pool);
 253         nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink);
 254         kbase_mem_pool_unlock(pool);
 255
 256         return nr_freed;
 257 }
 258
 259 static size_t kbase_mem_pool_grow(struct kbase_mem_pool *pool,
 260                 size_t nr_to_grow)
 261 {
 262         struct page *p;
 263         size_t i;
 264
 265         for (i = 0; i < nr_to_grow && !kbase_mem_pool_is_full(pool); i++) {
 266                 p = kbase_mem_pool_alloc_page(pool);
 267                 if (!p)
 268                         break;
 269                 kbase_mem_pool_add(pool, p);
 270         }
 271
 272         return i;
 273 }
 274
 275 size_t kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size)
 276 {
 277         size_t cur_size;
 278
 279         cur_size = kbase_mem_pool_size(pool);
 280
 281         if (new_size < cur_size)
 282                 kbase_mem_pool_shrink(pool, cur_size - new_size);
 283         else if (new_size > cur_size)
 284                 kbase_mem_pool_grow(pool, new_size - cur_size);
 285
 286         cur_size = kbase_mem_pool_size(pool);
 287
 288         return cur_size;
 289 }
 290
 291 void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size)
 292 {
 293         size_t cur_size;
 294         size_t nr_to_shrink;
 295
 296         kbase_mem_pool_lock(pool);
 297
 298         pool->max_size = max_size;
 299
 300         cur_size = kbase_mem_pool_size(pool);
 301         if (max_size < cur_size) {
 302                 nr_to_shrink = cur_size - max_size;
 303                 kbase_mem_pool_shrink_locked(pool, nr_to_shrink);
 304         }
 305
 306         kbase_mem_pool_unlock(pool);
 307 }
 308
 309
 310 static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s,
 311                 struct shrink_control *sc)
 312 {
 313         struct kbase_mem_pool *pool;
 314
 315         pool = container_of(s, struct kbase_mem_pool, reclaim);
 316         pool_dbg(pool, "reclaim count: %zu\n", kbase_mem_pool_size(pool));
 317         return kbase_mem_pool_size(pool);
 318 }
 319
 320 static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s,
 321                 struct shrink_control *sc)
 322 {
 323         struct kbase_mem_pool *pool;
 324         unsigned long freed;
 325
 326         pool = container_of(s, struct kbase_mem_pool, reclaim);
 327
 328         pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan);
 329
 330         freed = kbase_mem_pool_shrink(pool, sc->nr_to_scan);
 331
 332         pool_dbg(pool, "reclaim freed %ld pages\n", freed);
 333
 334         return freed;
 335 }
 336
 337 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 338 static int kbase_mem_pool_reclaim_shrink(struct shrinker *s,
 339                 struct shrink_control *sc)
 340 {
 341         if (sc->nr_to_scan == 0)
 342                 return kbase_mem_pool_reclaim_count_objects(s, sc);
 343
 344         return kbase_mem_pool_reclaim_scan_objects(s, sc);
 345 }
 346 #endif
 347
 348 int kbase_mem_pool_init(struct kbase_mem_pool *pool,
 349                 size_t max_size,
 350                 struct kbase_device *kbdev,
 351                 struct kbase_mem_pool *next_pool)
 352 {
 353         pool->cur_size = 0;
 354         pool->max_size = max_size;
 355         pool->kbdev = kbdev;
 356         pool->next_pool = next_pool;
 357
 358         spin_lock_init(&pool->pool_lock);
 359         INIT_LIST_HEAD(&pool->page_list);
 360
 361         /* Register shrinker */
 362 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
 363         pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink;
 364 #else
 365         pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects;
 366         pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects;
 367 #endif
 368         pool->reclaim.seeks = DEFAULT_SEEKS;
 369         /* Kernel versions prior to 3.1 :
 370          * struct shrinker does not define batch */
 371 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0)
 372         pool->reclaim.batch = 0;
 373 #endif
 374         register_shrinker(&pool->reclaim);
 375
 376         pool_dbg(pool, "initialized\n");
 377
 378         return 0;
 379 }
 380
 381 void kbase_mem_pool_term(struct kbase_mem_pool *pool)
 382 {
 383         struct kbase_mem_pool *next_pool = pool->next_pool;
 384         struct page *p;
 385         size_t nr_to_spill = 0;
 386         LIST_HEAD(spill_list);
 387         int i;
 388
 389         pool_dbg(pool, "terminate()\n");
 390
 391         unregister_shrinker(&pool->reclaim);
 392
 393         kbase_mem_pool_lock(pool);
 394         pool->max_size = 0;
 395
 396         if (next_pool && !kbase_mem_pool_is_full(next_pool)) {
 397                 /* Spill to next pool (may overspill) */
 398                 nr_to_spill = kbase_mem_pool_capacity(next_pool);
 399                 nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill);
 400
 401                 /* Zero pages first without holding the next_pool lock */
 402                 for (i = 0; i < nr_to_spill; i++) {
 403                         p = kbase_mem_pool_remove_locked(pool);
 404                         kbase_mem_pool_zero_page(pool, p);
 405                         list_add(&p->lru, &spill_list);
 406                 }
 407         }
 408
 409         while (!kbase_mem_pool_is_empty(pool)) {
 410                 /* Free remaining pages to kernel */
 411                 p = kbase_mem_pool_remove_locked(pool);
 412                 kbase_mem_pool_free_page(pool, p);
 413         }
 414
 415         kbase_mem_pool_unlock(pool);
 416
 417         if (next_pool && nr_to_spill) {
 418                 /* Add new page list to next_pool */
 419                 kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill);
 420
 421                 pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill);
 422         }
 423
 424         pool_dbg(pool, "terminated\n");
 425 }
 426
 427 struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool)
 428 {
 429         struct page *p;
 430
 431         pool_dbg(pool, "alloc()\n");
 432
 433         p = kbase_mem_pool_remove(pool);
 434
 435         if (!p && pool->next_pool) {
 436                 /* Allocate via next pool */
 437                 return kbase_mem_pool_alloc(pool->next_pool);
 438         }
 439
 440         if (!p) {
 441                 /* Get page from kernel */
 442                 p = kbase_mem_pool_alloc_page(pool);
 443         }
 444
 445         return p;
 446 }
 447
 448 void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p,
 449                 bool dirty)
 450 {
 451         struct kbase_mem_pool *next_pool = pool->next_pool;
 452
 453         pool_dbg(pool, "free()\n");
 454
 455         if (!kbase_mem_pool_is_full(pool)) {
 456                 /* Add to our own pool */
 457                 if (dirty)
 458                         kbase_mem_pool_sync_page(pool, p);
 459
 460                 kbase_mem_pool_add(pool, p);
 461         } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) {
 462                 /* Spill to next pool */
 463                 kbase_mem_pool_spill(next_pool, p);
 464         } else {
 465                 /* Free page */
 466                 kbase_mem_pool_free_page(pool, p);
 467         }
 468 }
 469
 470 int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages,
 471                 phys_addr_t *pages)
 472 {
 473         struct page *p;
 474         size_t nr_from_pool;
 475         size_t i;
 476         int err = -ENOMEM;
 477
 478         pool_dbg(pool, "alloc_pages(%zu):\n", nr_pages);
 479
 480         /* Get pages from this pool */
 481         kbase_mem_pool_lock(pool);
 482         nr_from_pool = min(nr_pages, kbase_mem_pool_size(pool));
 483         for (i = 0; i < nr_from_pool; i++) {
 484                 p = kbase_mem_pool_remove_locked(pool);
 485                 pages[i] = page_to_phys(p);
 486         }
 487         kbase_mem_pool_unlock(pool);
 488
 489         if (i != nr_pages && pool->next_pool) {
 490                 /* Allocate via next pool */
 491                 err = kbase_mem_pool_alloc_pages(pool->next_pool,
 492                                 nr_pages - i, pages + i);
 493
 494                 if (err)
 495                         goto err_rollback;
 496
 497                 i += nr_pages - i;
 498         }
 499
 500         /* Get any remaining pages from kernel */
 501         for (; i < nr_pages; i++) {
 502                 p = kbase_mem_pool_alloc_page(pool);
 503                 if (!p)
 504                         goto err_rollback;
 505                 pages[i] = page_to_phys(p);
 506         }
 507
 508         pool_dbg(pool, "alloc_pages(%zu) done\n", nr_pages);
 509
 510         return 0;
 511
 512 err_rollback:
 513         kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED);
 514         return err;
 515 }
 516
 517 static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool,
 518                 size_t nr_pages, phys_addr_t *pages, bool zero, bool sync)
 519 {
 520         struct page *p;
 521         size_t nr_to_pool = 0;
 522         LIST_HEAD(new_page_list);
 523         size_t i;
 524
 525         if (!nr_pages)
 526                 return;
 527
 528         pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n",
 529                         nr_pages, zero, sync);
 530
 531         /* Zero/sync pages first without holding the pool lock */
 532         for (i = 0; i < nr_pages; i++) {
 533                 if (unlikely(!pages[i]))
 534                         continue;
 535
 536                 p = phys_to_page(pages[i]);
 537
 538                 if (zero)
 539                         kbase_mem_pool_zero_page(pool, p);
 540                 else if (sync)
 541                         kbase_mem_pool_sync_page(pool, p);
 542
 543                 list_add(&p->lru, &new_page_list);
 544                 nr_to_pool++;
 545                 pages[i] = 0;
 546         }
 547
 548         /* Add new page list to pool */
 549         kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool);
 550
 551         pool_dbg(pool, "add_array(%zu) added %zu pages\n",
 552                         nr_pages, nr_to_pool);
 553 }
 554
 555 void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages,
 556                 phys_addr_t *pages, bool dirty, bool reclaimed)
 557 {
 558         struct kbase_mem_pool *next_pool = pool->next_pool;
 559         struct page *p;
 560         size_t nr_to_pool;
 561         LIST_HEAD(to_pool_list);
 562         size_t i = 0;
 563
 564         pool_dbg(pool, "free_pages(%zu):\n", nr_pages);
 565
 566         if (!reclaimed) {
 567                 /* Add to this pool */
 568                 nr_to_pool = kbase_mem_pool_capacity(pool);
 569                 nr_to_pool = min(nr_pages, nr_to_pool);
 570
 571                 kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty);
 572
 573                 i += nr_to_pool;
 574
 575                 if (i != nr_pages && next_pool) {
 576                         /* Spill to next pool (may overspill) */
 577                         nr_to_pool = kbase_mem_pool_capacity(next_pool);
 578                         nr_to_pool = min(nr_pages - i, nr_to_pool);
 579
 580                         kbase_mem_pool_add_array(next_pool, nr_to_pool,
 581                                         pages + i, true, dirty);
 582                         i += nr_to_pool;
 583                 }
 584         }
 585
 586         /* Free any remaining pages to kernel */
 587         for (; i < nr_pages; i++) {
 588                 if (unlikely(!pages[i]))
 589                         continue;
 590
 591                 p = phys_to_page(pages[i]);
 592                 if (reclaimed)
 593                         zone_page_state_add(-1, page_zone(p),
 594                                         NR_SLAB_RECLAIMABLE);
 595
 596                 kbase_mem_pool_free_page(pool, p);
 597                 pages[i] = 0;
 598         }
 599
 600         pool_dbg(pool, "free_pages(%zu) done\n", nr_pages);
 601 }