mm/gup: move gup_must_unshare() to mm/internal.h

This function is only used in gup.c and closely related.  It touches
FOLL_PIN so it must be moved before the next patch.

Link: https://lkml.kernel.org/r/12-v2-987e91b59705+36b-gup_tidy_jgg@nvidia.com
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Claudio Imbrenda <imbrenda@linux.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Mike Rapoport (IBM) <rppt@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index a0d5964..4a0695e 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3181,71 +3181,6 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
 }
 
 /*
- * Indicates for which pages that are write-protected in the page table,
- * whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the
- * GUP pin will remain consistent with the pages mapped into the page tables
- * of the MM.
- *
- * Temporary unmapping of PageAnonExclusive() pages or clearing of
- * PageAnonExclusive() has to protect against concurrent GUP:
- * * Ordinary GUP: Using the PT lock
- * * GUP-fast and fork(): mm->write_protect_seq
- * * GUP-fast and KSM or temporary unmapping (swap, migration): see
- *    page_try_share_anon_rmap()
- *
- * Must be called with the (sub)page that's actually referenced via the
- * page table entry, which might not necessarily be the head page for a
- * PTE-mapped THP.
- *
- * If the vma is NULL, we're coming from the GUP-fast path and might have
- * to fallback to the slow path just to lookup the vma.
- */
-static inline bool gup_must_unshare(struct vm_area_struct *vma,
-                                   unsigned int flags, struct page *page)
-{
-       /*
-        * FOLL_WRITE is implicitly handled correctly as the page table entry
-        * has to be writable -- and if it references (part of) an anonymous
-        * folio, that part is required to be marked exclusive.
-        */
-       if ((flags & (FOLL_WRITE | FOLL_PIN)) != FOLL_PIN)
-               return false;
-       /*
-        * Note: PageAnon(page) is stable until the page is actually getting
-        * freed.
-        */
-       if (!PageAnon(page)) {
-               /*
-                * We only care about R/O long-term pining: R/O short-term
-                * pinning does not have the semantics to observe successive
-                * changes through the process page tables.
-                */
-               if (!(flags & FOLL_LONGTERM))
-                       return false;
-
-               /* We really need the vma ... */
-               if (!vma)
-                       return true;
-
-               /*
-                * ... because we only care about writable private ("COW")
-                * mappings where we have to break COW early.
-                */
-               return is_cow_mapping(vma->vm_flags);
-       }
-
-       /* Paired with a memory barrier in page_try_share_anon_rmap(). */
-       if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
-               smp_rmb();
-
-       /*
-        * Note that PageKsm() pages cannot be exclusive, and consequently,
-        * cannot get pinned.
-        */
-       return !PageAnonExclusive(page);
-}
-
-/*
  * Indicates whether GUP can follow a PROT_NONE mapped page, or whether
  * a (NUMA hinting) fault is required.
  */

diff --git a/mm/internal.h b/mm/internal.h
index 3f75763..4f5ca34 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -858,6 +858,71 @@ int migrate_device_coherent_page(struct page *page);
 struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags);
 int __must_check try_grab_page(struct page *page, unsigned int flags);
 
+/*
+ * Indicates for which pages that are write-protected in the page table,
+ * whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the
+ * GUP pin will remain consistent with the pages mapped into the page tables
+ * of the MM.
+ *
+ * Temporary unmapping of PageAnonExclusive() pages or clearing of
+ * PageAnonExclusive() has to protect against concurrent GUP:
+ * * Ordinary GUP: Using the PT lock
+ * * GUP-fast and fork(): mm->write_protect_seq
+ * * GUP-fast and KSM or temporary unmapping (swap, migration): see
+ *    page_try_share_anon_rmap()
+ *
+ * Must be called with the (sub)page that's actually referenced via the
+ * page table entry, which might not necessarily be the head page for a
+ * PTE-mapped THP.
+ *
+ * If the vma is NULL, we're coming from the GUP-fast path and might have
+ * to fallback to the slow path just to lookup the vma.
+ */
+static inline bool gup_must_unshare(struct vm_area_struct *vma,
+                                   unsigned int flags, struct page *page)
+{
+       /*
+        * FOLL_WRITE is implicitly handled correctly as the page table entry
+        * has to be writable -- and if it references (part of) an anonymous
+        * folio, that part is required to be marked exclusive.
+        */
+       if ((flags & (FOLL_WRITE | FOLL_PIN)) != FOLL_PIN)
+               return false;
+       /*
+        * Note: PageAnon(page) is stable until the page is actually getting
+        * freed.
+        */
+       if (!PageAnon(page)) {
+               /*
+                * We only care about R/O long-term pining: R/O short-term
+                * pinning does not have the semantics to observe successive
+                * changes through the process page tables.
+                */
+               if (!(flags & FOLL_LONGTERM))
+                       return false;
+
+               /* We really need the vma ... */
+               if (!vma)
+                       return true;
+
+               /*
+                * ... because we only care about writable private ("COW")
+                * mappings where we have to break COW early.
+                */
+               return is_cow_mapping(vma->vm_flags);
+       }
+
+       /* Paired with a memory barrier in page_try_share_anon_rmap(). */
+       if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
+               smp_rmb();
+
+       /*
+        * Note that PageKsm() pages cannot be exclusive, and consequently,
+        * cannot get pinned.
+        */
+       return !PageAnonExclusive(page);
+}
+
 extern bool mirrored_kernelcore;
 
 static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)