drm/ttm: add transparent huge page support for wc or uc allocations v2

Add a new huge page pool and try to allocate from it when it makes sense.

v2: avoid compound pages for now

Signed-off-by: Christian König <christian.koenig@amd.com>
Acked-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index 3974732..b6f16e7 100644 (file)
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -95,7 +95,7 @@ struct ttm_pool_opts {
        unsigned        small;
 };
 
-#define NUM_POOLS 4
+#define NUM_POOLS 6
 
 /**
  * struct ttm_pool_manager - Holds memory pools for fst allocation
@@ -122,6 +122,8 @@ struct ttm_pool_manager {
                        struct ttm_page_pool    uc_pool;
                        struct ttm_page_pool    wc_pool_dma32;
                        struct ttm_page_pool    uc_pool_dma32;
+                       struct ttm_page_pool    wc_pool_huge;
+                       struct ttm_page_pool    uc_pool_huge;
                } ;
        };
 };
@@ -256,8 +258,8 @@ static int set_pages_array_uc(struct page **pages, int addrinarray)
 
 /**
  * Select the right pool or requested caching state and ttm flags. */
-static struct ttm_page_pool *ttm_get_pool(int flags,
-               enum ttm_caching_state cstate)
+static struct ttm_page_pool *ttm_get_pool(int flags, bool huge,
+                                         enum ttm_caching_state cstate)
 {
        int pool_index;
 
@@ -269,9 +271,15 @@ static struct ttm_page_pool *ttm_get_pool(int flags,
        else
                pool_index = 0x1;
 
-       if (flags & TTM_PAGE_FLAG_DMA32)
+       if (flags & TTM_PAGE_FLAG_DMA32) {
+               if (huge)
+                       return NULL;
                pool_index |= 0x2;
 
+       } else if (huge) {
+               pool_index |= 0x4;
+       }
+
        return &_manager->pools[pool_index];
 }
 
@@ -494,12 +502,14 @@ static void ttm_handle_caching_state_failure(struct list_head *pages,
  * pages returned in pages array.
  */
 static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
-               int ttm_flags, enum ttm_caching_state cstate, unsigned count)
+                              int ttm_flags, enum ttm_caching_state cstate,
+                              unsigned count, unsigned order)
 {
        struct page **caching_array;
        struct page *p;
        int r = 0;
-       unsigned i, cpages;
+       unsigned i, j, cpages;
+       unsigned npages = 1 << order;
        unsigned max_cpages = min(count,
                        (unsigned)(PAGE_SIZE/sizeof(struct page *)));
 
@@ -512,7 +522,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
        }
 
        for (i = 0, cpages = 0; i < count; ++i) {
-               p = alloc_page(gfp_flags);
+               p = alloc_pages(gfp_flags, order);
 
                if (!p) {
                        pr_err("Unable to get page %u\n", i);
@@ -531,14 +541,18 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
                        goto out;
                }
 
+               list_add(&p->lru, pages);
+
 #ifdef CONFIG_HIGHMEM
                /* gfp flags of highmem page should never be dma32 so we
                 * we should be fine in such case
                 */
-               if (!PageHighMem(p))
+               if (PageHighMem(p))
+                       continue;
+
 #endif
-               {
-                       caching_array[cpages++] = p;
+               for (j = 0; j < npages; ++j) {
+                       caching_array[cpages++] = p++;
                        if (cpages == max_cpages) {
 
                                r = ttm_set_pages_caching(caching_array,
@@ -552,8 +566,6 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
                                cpages = 0;
                        }
                }
-
-               list_add(&p->lru, pages);
        }
 
        if (cpages) {
@@ -573,9 +585,9 @@ out:
  * Fill the given pool if there aren't enough pages and the requested number of
  * pages is small.
  */
-static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
-               int ttm_flags, enum ttm_caching_state cstate, unsigned count,
-               unsigned long *irq_flags)
+static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags,
+                                     enum ttm_caching_state cstate,
+                                     unsigned count, unsigned long *irq_flags)
 {
        struct page *p;
        int r;
@@ -605,7 +617,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 
                INIT_LIST_HEAD(&new_pages);
                r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
-                               cstate, alloc_size);
+                                       cstate, alloc_size, 0);
                spin_lock_irqsave(&pool->lock, *irq_flags);
 
                if (!r) {
@@ -635,7 +647,7 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
                                   struct list_head *pages,
                                   int ttm_flags,
                                   enum ttm_caching_state cstate,
-                                  unsigned count)
+                                  unsigned count, unsigned order)
 {
        unsigned long irq_flags;
        struct list_head *p;
@@ -643,7 +655,9 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
        int r = 0;
 
        spin_lock_irqsave(&pool->lock, irq_flags);
-       ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
+       if (!order)
+               ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count,
+                                         &irq_flags);
 
        if (count >= pool->npages) {
                /* take all pages from the pool */
@@ -698,7 +712,7 @@ out:
                 * multiple requests in parallel.
                 **/
                r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate,
-                                       count);
+                                       count, order);
        }
 
        return r;
@@ -708,8 +722,9 @@ out:
 static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
                          enum ttm_caching_state cstate)
 {
+       struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
+       struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
        unsigned long irq_flags;
-       struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
        unsigned i;
 
        if (pool == NULL) {
@@ -737,8 +752,48 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
                return;
        }
 
+       i = 0;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+       if (huge) {
+               unsigned max_size, n2free;
+
+               spin_lock_irqsave(&huge->lock, irq_flags);
+               while (i < npages) {
+                       struct page *p = pages[i];
+                       unsigned j;
+
+                       if (!p)
+                               break;
+
+                       for (j = 0; j < HPAGE_PMD_NR; ++j)
+                               if (p++ != pages[i + j])
+                                   break;
+
+                       if (j != HPAGE_PMD_NR)
+                               break;
+
+                       list_add_tail(&pages[i]->lru, &huge->list);
+
+                       for (j = 0; j < HPAGE_PMD_NR; ++j)
+                               pages[i++] = NULL;
+                       huge->npages++;
+               }
+
+               /* Check that we don't go over the pool limit */
+               max_size = _manager->options.max_size;
+               max_size /= HPAGE_PMD_NR;
+               if (huge->npages > max_size)
+                       n2free = huge->npages - max_size;
+               else
+                       n2free = 0;
+               spin_unlock_irqrestore(&huge->lock, irq_flags);
+               if (n2free)
+                       ttm_page_pool_free(huge, n2free, false);
+       }
+#endif
+
        spin_lock_irqsave(&pool->lock, irq_flags);
-       for (i = 0; i < npages; i++) {
+       while (i < npages) {
                if (pages[i]) {
                        if (page_count(pages[i]) != 1)
                                pr_err("Erroneous page count. Leaking pages.\n");
@@ -746,6 +801,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
                        pages[i] = NULL;
                        pool->npages++;
                }
+               ++i;
        }
        /* Check that we don't go over the pool limit */
        npages = 0;
@@ -768,7 +824,8 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
 static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
                         enum ttm_caching_state cstate)
 {
-       struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
+       struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
+       struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
        struct list_head plist;
        struct page *p = NULL;
        unsigned count;
@@ -821,11 +878,28 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
                return 0;
        }
 
-       /* First we take pages from the pool */
+       count = 0;
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+       if (huge && npages >= HPAGE_PMD_NR) {
+               INIT_LIST_HEAD(&plist);
+               ttm_page_pool_get_pages(huge, &plist, flags, cstate,
+                                       npages / HPAGE_PMD_NR,
+                                       HPAGE_PMD_ORDER);
+
+               list_for_each_entry(p, &plist, lru) {
+                       unsigned j;
+
+                       for (j = 0; j < HPAGE_PMD_NR; ++j)
+                               pages[count++] = &p[j];
+               }
+       }
+#endif
+
        INIT_LIST_HEAD(&plist);
-       r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
+       r = ttm_page_pool_get_pages(pool, &plist, flags, cstate,
+                                   npages - count, 0);
 
-       count = 0;
        list_for_each_entry(p, &plist, lru)
                pages[count++] = p;
 
@@ -872,6 +946,14 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
        ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
                                  GFP_USER | GFP_DMA32, "uc dma");
 
+       ttm_page_pool_init_locked(&_manager->wc_pool_huge,
+                                 GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP),
+                                 "wc huge");
+
+       ttm_page_pool_init_locked(&_manager->uc_pool_huge,
+                                 GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP)
+                                 , "uc huge");
+
        _manager->options.max_size = max_pages;
        _manager->options.small = SMALL_ALLOCATION;
        _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
@@ -1041,12 +1123,12 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
                seq_printf(m, "No pool allocator running.\n");
                return 0;
        }
-       seq_printf(m, "%6s %12s %13s %8s\n",
+       seq_printf(m, "%7s %12s %13s %8s\n",
                        h[0], h[1], h[2], h[3]);
        for (i = 0; i < NUM_POOLS; ++i) {
                p = &_manager->pools[i];
 
-               seq_printf(m, "%6s %12ld %13ld %8d\n",
+               seq_printf(m, "%7s %12ld %13ld %8d\n",
                                p->name, p->nrefills,
                                p->nfrees, p->npages);
        }