2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
7 #include <linux/pagevec.h>
8 #include <linux/swap.h>
10 #include "gem/i915_gem_region.h"
12 #include "i915_gemfs.h"
13 #include "i915_gem_object.h"
14 #include "i915_scatterlist.h"
15 #include "i915_trace.h"
18 * Move pages to appropriate lru and release the pagevec, decrementing the
19 * ref count of those pages.
21 static void check_release_pagevec(struct pagevec *pvec)
23 check_move_unevictable_pages(pvec);
24 __pagevec_release(pvec);
28 static int shmem_get_pages(struct drm_i915_gem_object *obj)
30 struct drm_i915_private *i915 = to_i915(obj->base.dev);
31 struct intel_memory_region *mem = obj->mm.region;
32 const unsigned long page_count = obj->base.size / PAGE_SIZE;
34 struct address_space *mapping;
36 struct scatterlist *sg;
37 struct sgt_iter sgt_iter;
39 unsigned long last_pfn = 0; /* suppress gcc warning */
40 unsigned int max_segment = i915_sg_segment_size();
41 unsigned int sg_page_sizes;
46 * Assert that the object is not currently in any GPU domain. As it
47 * wasn't in the GTT, there shouldn't be any way it could have been in
50 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
51 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
54 * If there's no chance of allocating enough pages for the whole
57 if (obj->base.size > resource_size(&mem->region))
60 st = kmalloc(sizeof(*st), GFP_KERNEL);
65 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
71 * Get the list of pages out of our struct file. They'll be pinned
72 * at this point until we release them.
74 * Fail silently without starting the shrinker
76 mapping = obj->base.filp->f_mapping;
77 mapping_set_unevictable(mapping);
78 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
79 noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
84 for (i = 0; i < page_count; i++) {
85 const unsigned int shrink[] = {
86 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,
89 gfp_t gfp = noreclaim;
93 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
102 i915_gem_shrink(i915, 2 * page_count, NULL, *s++);
105 * We've tried hard to allocate the memory by reaping
106 * our own buffer, now let the real VM do its job and
107 * go down in flames if truly OOM.
109 * However, since graphics tend to be disposable,
110 * defer the oom here by reporting the ENOMEM back
114 /* reclaim and warn, but no oom */
115 gfp = mapping_gfp_mask(mapping);
118 * Our bo are always dirty and so we require
119 * kswapd to reclaim our pages (direct reclaim
120 * does not effectively begin pageout of our
121 * buffers on its own). However, direct reclaim
122 * only waits for kswapd when under allocation
123 * congestion. So as a result __GFP_RECLAIM is
124 * unreliable and fails to actually reclaim our
125 * dirty pages -- unless you try over and over
126 * again with !__GFP_NORETRY. However, we still
127 * want to fail this allocation rather than
128 * trigger the out-of-memory killer and for
129 * this we want __GFP_RETRY_MAYFAIL.
131 gfp |= __GFP_RETRY_MAYFAIL;
136 sg->length >= max_segment ||
137 page_to_pfn(page) != last_pfn + 1) {
139 sg_page_sizes |= sg->length;
143 sg_set_page(sg, page, PAGE_SIZE, 0);
145 sg->length += PAGE_SIZE;
147 last_pfn = page_to_pfn(page);
149 /* Check that the i965g/gm workaround works. */
150 GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL);
152 if (sg) { /* loop terminated early; short sg table */
153 sg_page_sizes |= sg->length;
157 /* Trim unused sg entries to avoid wasting memory. */
160 ret = i915_gem_gtt_prepare_pages(obj, st);
163 * DMA remapping failed? One possible cause is that
164 * it could not reserve enough large entries, asking
165 * for PAGE_SIZE chunks instead may be helpful.
167 if (max_segment > PAGE_SIZE) {
168 for_each_sgt_page(page, sgt_iter, st)
172 max_segment = PAGE_SIZE;
175 dev_warn(i915->drm.dev,
176 "Failed to DMA remap %lu pages\n",
182 if (i915_gem_object_needs_bit17_swizzle(obj))
183 i915_gem_object_do_bit_17_swizzle(obj, st);
185 __i915_gem_object_set_pages(obj, st, sg_page_sizes);
192 mapping_clear_unevictable(mapping);
197 for_each_sgt_page(page, sgt_iter, st) {
198 if (!pagevec_add(&pvec, page))
199 check_release_pagevec(&pvec);
201 if (pagevec_count(&pvec))
202 check_release_pagevec(&pvec);
208 * shmemfs first checks if there is enough memory to allocate the page
209 * and reports ENOSPC should there be insufficient, along with the usual
210 * ENOMEM for a genuine allocation failure.
212 * We use ENOSPC in our driver to mean that we have run out of aperture
213 * space and so want to translate the error from shmemfs back to our
214 * usual understanding of ENOMEM.
223 shmem_truncate(struct drm_i915_gem_object *obj)
226 * Our goal here is to return as much of the memory as
227 * is possible back to the system as we are called from OOM.
228 * To do this we must instruct the shmfs to drop all of its
229 * backing pages, *now*.
231 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
232 obj->mm.madv = __I915_MADV_PURGED;
233 obj->mm.pages = ERR_PTR(-EFAULT);
237 shmem_writeback(struct drm_i915_gem_object *obj)
239 struct address_space *mapping;
240 struct writeback_control wbc = {
241 .sync_mode = WB_SYNC_NONE,
242 .nr_to_write = SWAP_CLUSTER_MAX,
244 .range_end = LLONG_MAX,
250 * Leave mmapings intact (GTT will have been revoked on unbinding,
251 * leaving only CPU mmapings around) and add those pages to the LRU
252 * instead of invoking writeback so they are aged and paged out
255 mapping = obj->base.filp->f_mapping;
257 /* Begin writeback on each dirty page */
258 for (i = 0; i < obj->base.size >> PAGE_SHIFT; i++) {
261 page = find_lock_page(mapping, i);
265 if (!page_mapped(page) && clear_page_dirty_for_io(page)) {
268 SetPageReclaim(page);
269 ret = mapping->a_ops->writepage(page, &wbc);
270 if (!PageWriteback(page))
271 ClearPageReclaim(page);
282 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
283 struct sg_table *pages,
286 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
288 if (obj->mm.madv == I915_MADV_DONTNEED)
289 obj->mm.dirty = false;
292 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
293 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
294 drm_clflush_sg(pages);
296 __start_cpu_write(obj);
300 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
302 struct sgt_iter sgt_iter;
306 __i915_gem_object_release_shmem(obj, pages, true);
308 i915_gem_gtt_finish_pages(obj, pages);
310 if (i915_gem_object_needs_bit17_swizzle(obj))
311 i915_gem_object_save_bit_17_swizzle(obj, pages);
313 mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping);
316 for_each_sgt_page(page, sgt_iter, pages) {
318 set_page_dirty(page);
320 if (obj->mm.madv == I915_MADV_WILLNEED)
321 mark_page_accessed(page);
323 if (!pagevec_add(&pvec, page))
324 check_release_pagevec(&pvec);
326 if (pagevec_count(&pvec))
327 check_release_pagevec(&pvec);
328 obj->mm.dirty = false;
330 sg_free_table(pages);
335 shmem_pwrite(struct drm_i915_gem_object *obj,
336 const struct drm_i915_gem_pwrite *arg)
338 struct address_space *mapping = obj->base.filp->f_mapping;
339 char __user *user_data = u64_to_user_ptr(arg->data_ptr);
343 /* Caller already validated user args */
344 GEM_BUG_ON(!access_ok(user_data, arg->size));
347 * Before we instantiate/pin the backing store for our use, we
348 * can prepopulate the shmemfs filp efficiently using a write into
349 * the pagecache. We avoid the penalty of instantiating all the
350 * pages, important if the user is just writing to a few and never
351 * uses the object on the GPU, and using a direct write into shmemfs
352 * allows it to avoid the cost of retrieving a page (either swapin
353 * or clearing-before-use) before it is overwritten.
355 if (i915_gem_object_has_pages(obj))
358 if (obj->mm.madv != I915_MADV_WILLNEED)
362 * Before the pages are instantiated the object is treated as being
363 * in the CPU domain. The pages will be clflushed as required before
364 * use, and we can freely write into the pages directly. If userspace
365 * races pwrite with any other operation; corruption will ensue -
366 * that is userspace's prerogative!
370 offset = arg->offset;
371 pg = offset_in_page(offset);
374 unsigned int len, unwritten;
380 len = PAGE_SIZE - pg;
384 /* Prefault the user page to reduce potential recursion */
385 err = __get_user(c, user_data);
389 err = __get_user(c, user_data + len - 1);
393 err = pagecache_write_begin(obj->base.filp, mapping,
399 vaddr = kmap_atomic(page);
400 unwritten = __copy_from_user_inatomic(vaddr + pg,
403 kunmap_atomic(vaddr);
405 err = pagecache_write_end(obj->base.filp, mapping,
406 offset, len, len - unwritten,
411 /* We don't handle -EFAULT, leave it to the caller to check */
424 static void shmem_release(struct drm_i915_gem_object *obj)
426 i915_gem_object_release_memory_region(obj);
428 fput(obj->base.filp);
431 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
432 .name = "i915_gem_object_shmem",
433 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
434 I915_GEM_OBJECT_IS_SHRINKABLE,
436 .get_pages = shmem_get_pages,
437 .put_pages = shmem_put_pages,
438 .truncate = shmem_truncate,
439 .writeback = shmem_writeback,
441 .pwrite = shmem_pwrite,
443 .release = shmem_release,
446 static int __create_shmem(struct drm_i915_private *i915,
447 struct drm_gem_object *obj,
448 resource_size_t size)
450 unsigned long flags = VM_NORESERVE;
453 drm_gem_private_object_init(&i915->drm, obj, size);
456 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
459 filp = shmem_file_setup("i915", size, flags);
461 return PTR_ERR(filp);
467 static int shmem_object_init(struct intel_memory_region *mem,
468 struct drm_i915_gem_object *obj,
469 resource_size_t size,
472 static struct lock_class_key lock_class;
473 struct drm_i915_private *i915 = mem->i915;
474 struct address_space *mapping;
475 unsigned int cache_level;
479 ret = __create_shmem(i915, &obj->base, size);
483 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
484 if (IS_I965GM(i915) || IS_I965G(i915)) {
485 /* 965gm cannot relocate objects above 4GiB. */
486 mask &= ~__GFP_HIGHMEM;
490 mapping = obj->base.filp->f_mapping;
491 mapping_set_gfp_mask(mapping, mask);
492 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
494 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class);
496 obj->write_domain = I915_GEM_DOMAIN_CPU;
497 obj->read_domains = I915_GEM_DOMAIN_CPU;
500 /* On some devices, we can have the GPU use the LLC (the CPU
501 * cache) for about a 10% performance improvement
502 * compared to uncached. Graphics requests other than
503 * display scanout are coherent with the CPU in
504 * accessing this cache. This means in this mode we
505 * don't need to clflush on the CPU side, and on the
506 * GPU side we only need to flush internal caches to
507 * get data visible to the CPU.
509 * However, we maintain the display planes as UC, and so
510 * need to rebind when first used as such.
512 cache_level = I915_CACHE_LLC;
514 cache_level = I915_CACHE_NONE;
516 i915_gem_object_set_cache_coherency(obj, cache_level);
518 i915_gem_object_init_memory_region(obj, mem, 0);
523 struct drm_i915_gem_object *
524 i915_gem_object_create_shmem(struct drm_i915_private *i915,
525 resource_size_t size)
527 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
531 /* Allocate a new GEM object and fill it with the supplied data */
532 struct drm_i915_gem_object *
533 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
534 const void *data, resource_size_t size)
536 struct drm_i915_gem_object *obj;
538 resource_size_t offset;
541 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE));
545 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
547 file = obj->base.filp;
550 unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
552 void *pgdata, *vaddr;
554 err = pagecache_write_begin(file, file->f_mapping,
561 memcpy(vaddr, data, len);
564 err = pagecache_write_end(file, file->f_mapping,
578 i915_gem_object_put(obj);
582 static int init_shmem(struct intel_memory_region *mem)
586 err = i915_gemfs_init(mem->i915);
588 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n",
592 intel_memory_region_set_name(mem, "system");
594 return 0; /* Don't error, we can simply fallback to the kernel mnt */
597 static void release_shmem(struct intel_memory_region *mem)
599 i915_gemfs_fini(mem->i915);
602 static const struct intel_memory_region_ops shmem_region_ops = {
604 .release = release_shmem,
605 .init_object = shmem_object_init,
608 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915)
610 return intel_memory_region_create(i915, 0,
611 totalram_pages() << PAGE_SHIFT,
616 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)
618 return obj->ops == &i915_gem_shmem_ops;