2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
7 #include <linux/pagevec.h>
8 #include <linux/shmem_fs.h>
9 #include <linux/swap.h>
11 #include <drm/drm_cache.h>
13 #include "gem/i915_gem_region.h"
15 #include "i915_gem_object.h"
16 #include "i915_gem_tiling.h"
17 #include "i915_gemfs.h"
18 #include "i915_scatterlist.h"
19 #include "i915_trace.h"
22 * Move folios to appropriate lru and release the batch, decrementing the
23 * ref count of those folios.
25 static void check_release_folio_batch(struct folio_batch *fbatch)
27 check_move_unevictable_folios(fbatch);
28 __folio_batch_release(fbatch);
32 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping,
33 bool dirty, bool backup)
35 struct sgt_iter sgt_iter;
36 struct folio_batch fbatch;
37 struct folio *last = NULL;
40 mapping_clear_unevictable(mapping);
42 folio_batch_init(&fbatch);
43 for_each_sgt_page(page, sgt_iter, st) {
44 struct folio *folio = page_folio(page);
50 folio_mark_dirty(folio);
52 folio_mark_accessed(folio);
54 if (!folio_batch_add(&fbatch, folio))
55 check_release_folio_batch(&fbatch);
58 check_release_folio_batch(&fbatch);
63 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st,
64 size_t size, struct intel_memory_region *mr,
65 struct address_space *mapping,
66 unsigned int max_segment)
68 unsigned int page_count; /* restricted by sg_alloc_table */
70 struct scatterlist *sg;
71 unsigned long next_pfn = 0; /* suppress gcc warning */
75 if (overflows_type(size / PAGE_SIZE, page_count))
78 page_count = size / PAGE_SIZE;
80 * If there's no chance of allocating enough pages for the whole
83 if (size > resource_size(&mr->region))
86 if (sg_alloc_table(st, page_count, GFP_KERNEL | __GFP_NOWARN))
90 * Get the list of pages out of our struct file. They'll be pinned
91 * at this point until we release them.
93 * Fail silently without starting the shrinker
95 mapping_set_unevictable(mapping);
96 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
97 noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
101 for (i = 0; i < page_count; i++) {
103 const unsigned int shrink[] = {
104 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,
107 gfp_t gfp = noreclaim;
111 folio = shmem_read_folio_gfp(mapping, i, gfp);
116 ret = PTR_ERR(folio);
120 i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++);
123 * We've tried hard to allocate the memory by reaping
124 * our own buffer, now let the real VM do its job and
125 * go down in flames if truly OOM.
127 * However, since graphics tend to be disposable,
128 * defer the oom here by reporting the ENOMEM back
132 /* reclaim and warn, but no oom */
133 gfp = mapping_gfp_mask(mapping);
136 * Our bo are always dirty and so we require
137 * kswapd to reclaim our pages (direct reclaim
138 * does not effectively begin pageout of our
139 * buffers on its own). However, direct reclaim
140 * only waits for kswapd when under allocation
141 * congestion. So as a result __GFP_RECLAIM is
142 * unreliable and fails to actually reclaim our
143 * dirty pages -- unless you try over and over
144 * again with !__GFP_NORETRY. However, we still
145 * want to fail this allocation rather than
146 * trigger the out-of-memory killer and for
147 * this we want __GFP_RETRY_MAYFAIL.
149 gfp |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
154 sg->length >= max_segment ||
155 folio_pfn(folio) != next_pfn) {
160 sg_set_folio(sg, folio, folio_size(folio), 0);
162 /* XXX: could overflow? */
163 sg->length += folio_size(folio);
165 next_pfn = folio_pfn(folio) + folio_nr_pages(folio);
166 i += folio_nr_pages(folio) - 1;
168 /* Check that the i965g/gm workaround works. */
169 GEM_BUG_ON(gfp & __GFP_DMA32 && next_pfn >= 0x00100000UL);
171 if (sg) /* loop terminated early; short sg table */
174 /* Trim unused sg entries to avoid wasting memory. */
181 shmem_sg_free_table(st, mapping, false, false);
183 mapping_clear_unevictable(mapping);
188 * shmemfs first checks if there is enough memory to allocate the page
189 * and reports ENOSPC should there be insufficient, along with the usual
190 * ENOMEM for a genuine allocation failure.
192 * We use ENOSPC in our driver to mean that we have run out of aperture
193 * space and so want to translate the error from shmemfs back to our
194 * usual understanding of ENOMEM.
202 static int shmem_get_pages(struct drm_i915_gem_object *obj)
204 struct drm_i915_private *i915 = to_i915(obj->base.dev);
205 struct intel_memory_region *mem = obj->mm.region;
206 struct address_space *mapping = obj->base.filp->f_mapping;
207 unsigned int max_segment = i915_sg_segment_size(i915->drm.dev);
209 struct sgt_iter sgt_iter;
214 * Assert that the object is not currently in any GPU domain. As it
215 * wasn't in the GTT, there shouldn't be any way it could have been in
218 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
219 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
222 st = kmalloc(sizeof(*st), GFP_KERNEL | __GFP_NOWARN);
226 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping,
231 ret = i915_gem_gtt_prepare_pages(obj, st);
234 * DMA remapping failed? One possible cause is that
235 * it could not reserve enough large entries, asking
236 * for PAGE_SIZE chunks instead may be helpful.
238 if (max_segment > PAGE_SIZE) {
239 for_each_sgt_page(page, sgt_iter, st)
244 max_segment = PAGE_SIZE;
247 dev_warn(i915->drm.dev,
248 "Failed to DMA remap %zu pages\n",
249 obj->base.size >> PAGE_SHIFT);
254 if (i915_gem_object_needs_bit17_swizzle(obj))
255 i915_gem_object_do_bit_17_swizzle(obj, st);
257 if (i915_gem_object_can_bypass_llc(obj))
258 obj->cache_dirty = true;
260 __i915_gem_object_set_pages(obj, st);
265 shmem_sg_free_table(st, mapping, false, false);
267 * shmemfs first checks if there is enough memory to allocate the page
268 * and reports ENOSPC should there be insufficient, along with the usual
269 * ENOMEM for a genuine allocation failure.
271 * We use ENOSPC in our driver to mean that we have run out of aperture
272 * space and so want to translate the error from shmemfs back to our
273 * usual understanding of ENOMEM.
285 shmem_truncate(struct drm_i915_gem_object *obj)
288 * Our goal here is to return as much of the memory as
289 * is possible back to the system as we are called from OOM.
290 * To do this we must instruct the shmfs to drop all of its
291 * backing pages, *now*.
293 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
294 obj->mm.madv = __I915_MADV_PURGED;
295 obj->mm.pages = ERR_PTR(-EFAULT);
300 void __shmem_writeback(size_t size, struct address_space *mapping)
302 struct writeback_control wbc = {
303 .sync_mode = WB_SYNC_NONE,
304 .nr_to_write = SWAP_CLUSTER_MAX,
306 .range_end = LLONG_MAX,
312 * Leave mmapings intact (GTT will have been revoked on unbinding,
313 * leaving only CPU mmapings around) and add those pages to the LRU
314 * instead of invoking writeback so they are aged and paged out
318 /* Begin writeback on each dirty page */
319 for (i = 0; i < size >> PAGE_SHIFT; i++) {
322 page = find_lock_page(mapping, i);
326 if (!page_mapped(page) && clear_page_dirty_for_io(page)) {
329 SetPageReclaim(page);
330 ret = mapping->a_ops->writepage(page, &wbc);
331 if (!PageWriteback(page))
332 ClearPageReclaim(page);
343 shmem_writeback(struct drm_i915_gem_object *obj)
345 __shmem_writeback(obj->base.size, obj->base.filp->f_mapping);
348 static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags)
350 switch (obj->mm.madv) {
351 case I915_MADV_DONTNEED:
352 return i915_gem_object_truncate(obj);
353 case __I915_MADV_PURGED:
357 if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK)
358 shmem_writeback(obj);
364 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
365 struct sg_table *pages,
368 struct drm_i915_private *i915 = to_i915(obj->base.dev);
370 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
372 if (obj->mm.madv == I915_MADV_DONTNEED)
373 obj->mm.dirty = false;
376 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
377 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
378 drm_clflush_sg(pages);
380 __start_cpu_write(obj);
382 * On non-LLC igfx platforms, force the flush-on-acquire if this is ever
383 * swapped-in. Our async flush path is not trust worthy enough yet(and
384 * happens in the wrong order), and with some tricks it's conceivable
385 * for userspace to change the cache-level to I915_CACHE_NONE after the
386 * pages are swapped-in, and since execbuf binds the object before doing
387 * the async flush, we have a race window.
389 if (!HAS_LLC(i915) && !IS_DGFX(i915))
390 obj->cache_dirty = true;
393 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)
395 __i915_gem_object_release_shmem(obj, pages, true);
397 i915_gem_gtt_finish_pages(obj, pages);
399 if (i915_gem_object_needs_bit17_swizzle(obj))
400 i915_gem_object_save_bit_17_swizzle(obj, pages);
402 shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping,
403 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED);
405 obj->mm.dirty = false;
409 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
411 if (likely(i915_gem_object_has_struct_page(obj)))
412 i915_gem_object_put_pages_shmem(obj, pages);
414 i915_gem_object_put_pages_phys(obj, pages);
418 shmem_pwrite(struct drm_i915_gem_object *obj,
419 const struct drm_i915_gem_pwrite *arg)
421 struct address_space *mapping = obj->base.filp->f_mapping;
422 const struct address_space_operations *aops = mapping->a_ops;
423 char __user *user_data = u64_to_user_ptr(arg->data_ptr);
427 /* Caller already validated user args */
428 GEM_BUG_ON(!access_ok(user_data, arg->size));
430 if (!i915_gem_object_has_struct_page(obj))
431 return i915_gem_object_pwrite_phys(obj, arg);
434 * Before we instantiate/pin the backing store for our use, we
435 * can prepopulate the shmemfs filp efficiently using a write into
436 * the pagecache. We avoid the penalty of instantiating all the
437 * pages, important if the user is just writing to a few and never
438 * uses the object on the GPU, and using a direct write into shmemfs
439 * allows it to avoid the cost of retrieving a page (either swapin
440 * or clearing-before-use) before it is overwritten.
442 if (i915_gem_object_has_pages(obj))
445 if (obj->mm.madv != I915_MADV_WILLNEED)
449 * Before the pages are instantiated the object is treated as being
450 * in the CPU domain. The pages will be clflushed as required before
451 * use, and we can freely write into the pages directly. If userspace
452 * races pwrite with any other operation; corruption will ensue -
453 * that is userspace's prerogative!
457 offset = arg->offset;
458 pg = offset_in_page(offset);
461 unsigned int len, unwritten;
465 char __maybe_unused c;
467 len = PAGE_SIZE - pg;
471 /* Prefault the user page to reduce potential recursion */
472 err = __get_user(c, user_data);
476 err = __get_user(c, user_data + len - 1);
480 err = aops->write_begin(obj->base.filp, mapping, offset, len,
485 vaddr = kmap_atomic(page);
486 unwritten = __copy_from_user_inatomic(vaddr + pg,
489 kunmap_atomic(vaddr);
491 err = aops->write_end(obj->base.filp, mapping, offset, len,
492 len - unwritten, page, data);
496 /* We don't handle -EFAULT, leave it to the caller to check */
510 shmem_pread(struct drm_i915_gem_object *obj,
511 const struct drm_i915_gem_pread *arg)
513 if (!i915_gem_object_has_struct_page(obj))
514 return i915_gem_object_pread_phys(obj, arg);
519 static void shmem_release(struct drm_i915_gem_object *obj)
521 if (i915_gem_object_has_struct_page(obj))
522 i915_gem_object_release_memory_region(obj);
524 fput(obj->base.filp);
527 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
528 .name = "i915_gem_object_shmem",
529 .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
531 .get_pages = shmem_get_pages,
532 .put_pages = shmem_put_pages,
533 .truncate = shmem_truncate,
534 .shrink = shmem_shrink,
536 .pwrite = shmem_pwrite,
537 .pread = shmem_pread,
539 .release = shmem_release,
542 static int __create_shmem(struct drm_i915_private *i915,
543 struct drm_gem_object *obj,
544 resource_size_t size)
546 unsigned long flags = VM_NORESERVE;
549 drm_gem_private_object_init(&i915->drm, obj, size);
551 /* XXX: The __shmem_file_setup() function returns -EINVAL if size is
552 * greater than MAX_LFS_FILESIZE.
553 * To handle the same error as other code that returns -E2BIG when
554 * the size is too large, we add a code that returns -E2BIG when the
555 * size is larger than the size that can be handled.
556 * If BITS_PER_LONG is 32, size > MAX_LFS_FILESIZE is always false,
557 * so we only needs to check when BITS_PER_LONG is 64.
558 * If BITS_PER_LONG is 32, E2BIG checks are processed when
559 * i915_gem_object_size_2big() is called before init_object() callback
562 if (BITS_PER_LONG == 64 && size > MAX_LFS_FILESIZE)
566 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
569 filp = shmem_file_setup("i915", size, flags);
571 return PTR_ERR(filp);
577 static int shmem_object_init(struct intel_memory_region *mem,
578 struct drm_i915_gem_object *obj,
579 resource_size_t offset,
580 resource_size_t size,
581 resource_size_t page_size,
584 static struct lock_class_key lock_class;
585 struct drm_i915_private *i915 = mem->i915;
586 struct address_space *mapping;
587 unsigned int cache_level;
591 ret = __create_shmem(i915, &obj->base, size);
595 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
596 if (IS_I965GM(i915) || IS_I965G(i915)) {
597 /* 965gm cannot relocate objects above 4GiB. */
598 mask &= ~__GFP_HIGHMEM;
602 mapping = obj->base.filp->f_mapping;
603 mapping_set_gfp_mask(mapping, mask);
604 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
606 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, flags);
607 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
608 obj->write_domain = I915_GEM_DOMAIN_CPU;
609 obj->read_domains = I915_GEM_DOMAIN_CPU;
612 * MTL doesn't snoop CPU cache by default for GPU access (namely
613 * 1-way coherency). However some UMD's are currently depending on
614 * that. Make 1-way coherent the default setting for MTL. A follow
615 * up patch will extend the GEM_CREATE uAPI to allow UMD's specify
616 * caching mode at BO creation time
618 if (HAS_LLC(i915) || (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)))
619 /* On some devices, we can have the GPU use the LLC (the CPU
620 * cache) for about a 10% performance improvement
621 * compared to uncached. Graphics requests other than
622 * display scanout are coherent with the CPU in
623 * accessing this cache. This means in this mode we
624 * don't need to clflush on the CPU side, and on the
625 * GPU side we only need to flush internal caches to
626 * get data visible to the CPU.
628 * However, we maintain the display planes as UC, and so
629 * need to rebind when first used as such.
631 cache_level = I915_CACHE_LLC;
633 cache_level = I915_CACHE_NONE;
635 i915_gem_object_set_cache_coherency(obj, cache_level);
637 i915_gem_object_init_memory_region(obj, mem);
642 struct drm_i915_gem_object *
643 i915_gem_object_create_shmem(struct drm_i915_private *i915,
644 resource_size_t size)
646 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
650 /* Allocate a new GEM object and fill it with the supplied data */
651 struct drm_i915_gem_object *
652 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
653 const void *data, resource_size_t size)
655 struct drm_i915_gem_object *obj;
657 const struct address_space_operations *aops;
658 resource_size_t offset;
661 GEM_WARN_ON(IS_DGFX(dev_priv));
662 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE));
666 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
668 file = obj->base.filp;
669 aops = file->f_mapping->a_ops;
672 unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
674 void *pgdata, *vaddr;
676 err = aops->write_begin(file, file->f_mapping, offset, len,
682 memcpy(vaddr, data, len);
685 err = aops->write_end(file, file->f_mapping, offset, len, len,
698 i915_gem_object_put(obj);
702 static int init_shmem(struct intel_memory_region *mem)
704 i915_gemfs_init(mem->i915);
705 intel_memory_region_set_name(mem, "system");
707 return 0; /* We have fallback to the kernel mnt if gemfs init failed. */
710 static int release_shmem(struct intel_memory_region *mem)
712 i915_gemfs_fini(mem->i915);
716 static const struct intel_memory_region_ops shmem_region_ops = {
718 .release = release_shmem,
719 .init_object = shmem_object_init,
722 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915,
723 u16 type, u16 instance)
725 return intel_memory_region_create(i915, 0,
726 totalram_pages() << PAGE_SHIFT,
732 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)
734 return obj->ops == &i915_gem_shmem_ops;