mm/mmap: refactor locking out of __vma_adjust()

Move the locking into vma_prepare() and vma_complete() for use elsewhere

Link: https://lkml.kernel.org/r/20230120162650.984577-41-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/internal.h b/mm/internal.h
index ffd6524..90bb207 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -941,4 +941,18 @@ static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
 
        return 0;
 }
+
+/*
+ * VMA lock generalization
+ */
+struct vma_prepare {
+       struct vm_area_struct *vma;
+       struct vm_area_struct *adj_next;
+       struct file *file;
+       struct address_space *mapping;
+       struct anon_vma *anon_vma;
+       struct vm_area_struct *insert;
+       struct vm_area_struct *remove;
+       struct vm_area_struct *remove2;
+};
 #endif /* __MM_INTERNAL_H */

diff --git a/mm/mmap.c b/mm/mmap.c
index 12545ec..1ef284b 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -574,6 +574,127 @@ nomem:
 }
 
 /*
+ * vma_prepare() - Helper function for handling locking VMAs prior to altering
+ * @vp: The initialized vma_prepare struct
+ */
+static inline void vma_prepare(struct vma_prepare *vp)
+{
+       if (vp->file) {
+               uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
+
+               if (vp->adj_next)
+                       uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
+                                     vp->adj_next->vm_end);
+
+               i_mmap_lock_write(vp->mapping);
+               if (vp->insert && vp->insert->vm_file) {
+                       /*
+                        * Put into interval tree now, so instantiated pages
+                        * are visible to arm/parisc __flush_dcache_page
+                        * throughout; but we cannot insert into address
+                        * space until vma start or end is updated.
+                        */
+                       __vma_link_file(vp->insert,
+                                       vp->insert->vm_file->f_mapping);
+               }
+       }
+
+       if (vp->anon_vma) {
+               anon_vma_lock_write(vp->anon_vma);
+               anon_vma_interval_tree_pre_update_vma(vp->vma);
+               if (vp->adj_next)
+                       anon_vma_interval_tree_pre_update_vma(vp->adj_next);
+       }
+
+       if (vp->file) {
+               flush_dcache_mmap_lock(vp->mapping);
+               vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
+               if (vp->adj_next)
+                       vma_interval_tree_remove(vp->adj_next,
+                                                &vp->mapping->i_mmap);
+       }
+
+}
+
+/*
+ * vma_complete- Helper function for handling the unlocking after altering VMAs,
+ * or for inserting a VMA.
+ *
+ * @vp: The vma_prepare struct
+ * @vmi: The vma iterator
+ * @mm: The mm_struct
+ */
+static inline void vma_complete(struct vma_prepare *vp,
+                               struct vma_iterator *vmi, struct mm_struct *mm)
+{
+       if (vp->file) {
+               if (vp->adj_next)
+                       vma_interval_tree_insert(vp->adj_next,
+                                                &vp->mapping->i_mmap);
+               vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
+               flush_dcache_mmap_unlock(vp->mapping);
+       }
+
+       if (vp->remove && vp->file) {
+               __remove_shared_vm_struct(vp->remove, vp->file, vp->mapping);
+               if (vp->remove2)
+                       __remove_shared_vm_struct(vp->remove2, vp->file,
+                                                 vp->mapping);
+       } else if (vp->insert) {
+               /*
+                * split_vma has split insert from vma, and needs
+                * us to insert it before dropping the locks
+                * (it may either follow vma or precede it).
+                */
+               vma_iter_store(vmi, vp->insert);
+               mm->map_count++;
+       }
+
+       if (vp->anon_vma) {
+               anon_vma_interval_tree_post_update_vma(vp->vma);
+               if (vp->adj_next)
+                       anon_vma_interval_tree_post_update_vma(vp->adj_next);
+               anon_vma_unlock_write(vp->anon_vma);
+       }
+
+       if (vp->file) {
+               i_mmap_unlock_write(vp->mapping);
+               uprobe_mmap(vp->vma);
+
+               if (vp->adj_next)
+                       uprobe_mmap(vp->adj_next);
+       }
+
+       if (vp->remove) {
+again:
+               if (vp->file) {
+                       uprobe_munmap(vp->remove, vp->remove->vm_start,
+                                     vp->remove->vm_end);
+                       fput(vp->file);
+               }
+               if (vp->remove->anon_vma)
+                       anon_vma_merge(vp->vma, vp->remove);
+               mm->map_count--;
+               mpol_put(vma_policy(vp->remove));
+               if (!vp->remove2)
+                       WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
+               vm_area_free(vp->remove);
+
+               /*
+                * In mprotect's case 6 (see comments on vma_merge),
+                * we must remove next_next too.
+                */
+               if (vp->remove2) {
+                       vp->remove = vp->remove2;
+                       vp->remove2 = NULL;
+                       goto again;
+               }
+       }
+       if (vp->insert && vp->file)
+               uprobe_mmap(vp->insert);
+}
+
+/*
  * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
  * is already present in an i_mmap tree without adjusting the tree.
  * The following helper function should be used when such adjustments
@@ -588,14 +709,13 @@ int __vma_adjust(struct vma_iterator *vmi, struct vm_area_struct *vma,
        struct vm_area_struct *next_next = NULL;        /* uninit var warning */
        struct vm_area_struct *next = find_vma(mm, vma->vm_end);
        struct vm_area_struct *orig_vma = vma;
-       struct address_space *mapping = NULL;
-       struct rb_root_cached *root = NULL;
        struct anon_vma *anon_vma = NULL;
        struct file *file = vma->vm_file;
        bool vma_changed = false;
        long adjust_next = 0;
        int remove_next = 0;
        struct vm_area_struct *exporter = NULL, *importer = NULL;
+       struct vma_prepare vma_prep;
 
        if (next && !insert) {
                if (end >= next->vm_end) {
@@ -691,39 +811,22 @@ int __vma_adjust(struct vma_iterator *vmi, struct vm_area_struct *vma,
                           anon_vma != next->anon_vma);
 
        vma_adjust_trans_huge(orig_vma, start, end, adjust_next);
-       if (file) {
-               mapping = file->f_mapping;
-               root = &mapping->i_mmap;
-               uprobe_munmap(vma, vma->vm_start, vma->vm_end);
-
-               if (adjust_next)
-                       uprobe_munmap(next, next->vm_start, next->vm_end);
-
-               i_mmap_lock_write(mapping);
-               if (insert && insert->vm_file) {
-                       /*
-                        * Put into interval tree now, so instantiated pages
-                        * are visible to arm/parisc __flush_dcache_page
-                        * throughout; but we cannot insert into address
-                        * space until vma start or end is updated.
-                        */
-                       __vma_link_file(insert, insert->vm_file->f_mapping);
-               }
-       }
 
-       if (anon_vma) {
-               anon_vma_lock_write(anon_vma);
-               anon_vma_interval_tree_pre_update_vma(vma);
-               if (adjust_next)
-                       anon_vma_interval_tree_pre_update_vma(next);
+       memset(&vma_prep, 0, sizeof(vma_prep));
+       vma_prep.vma = vma;
+       vma_prep.anon_vma = anon_vma;
+       vma_prep.file = file;
+       if (adjust_next)
+               vma_prep.adj_next = next;
+       if (file)
+               vma_prep.mapping = file->f_mapping;
+       vma_prep.insert = insert;
+       if (remove_next) {
+               vma_prep.remove = next;
+               vma_prep.remove2 = next_next;
        }
 
-       if (file) {
-               flush_dcache_mmap_lock(mapping);
-               vma_interval_tree_remove(vma, root);
-               if (adjust_next)
-                       vma_interval_tree_remove(next, root);
-       }
+       vma_prepare(&vma_prep);
 
        if (start != vma->vm_start) {
                if (vma->vm_start < start) {
@@ -761,69 +864,7 @@ int __vma_adjust(struct vma_iterator *vmi, struct vm_area_struct *vma,
                vma_iter_store(vmi, next);
        }
 
-       if (file) {
-               if (adjust_next)
-                       vma_interval_tree_insert(next, root);
-               vma_interval_tree_insert(vma, root);
-               flush_dcache_mmap_unlock(mapping);
-       }
-
-       if (remove_next && file) {
-               __remove_shared_vm_struct(next, file, mapping);
-               if (remove_next == 2)
-                       __remove_shared_vm_struct(next_next, file, mapping);
-       } else if (insert) {
-               /*
-                * split_vma has split insert from vma, and needs
-                * us to insert it before dropping the locks
-                * (it may either follow vma or precede it).
-                */
-               vma_iter_store(vmi, insert);
-               mm->map_count++;
-       }
-
-       if (anon_vma) {
-               anon_vma_interval_tree_post_update_vma(vma);
-               if (adjust_next)
-                       anon_vma_interval_tree_post_update_vma(next);
-               anon_vma_unlock_write(anon_vma);
-       }
-
-       if (file) {
-               i_mmap_unlock_write(mapping);
-               uprobe_mmap(vma);
-
-               if (adjust_next)
-                       uprobe_mmap(next);
-       }
-
-       if (remove_next) {
-again:
-               if (file) {
-                       uprobe_munmap(next, next->vm_start, next->vm_end);
-                       fput(file);
-               }
-               if (next->anon_vma)
-                       anon_vma_merge(vma, next);
-               mm->map_count--;
-               mpol_put(vma_policy(next));
-               if (remove_next != 2)
-                       BUG_ON(vma->vm_end < next->vm_end);
-               vm_area_free(next);
-
-               /*
-                * In mprotect's case 6 (see comments on vma_merge),
-                * we must remove next_next too.
-                */
-               if (remove_next == 2) {
-                       remove_next = 1;
-                       next = next_next;
-                       goto again;
-               }
-       }
-       if (insert && file)
-               uprobe_mmap(insert);
-
+       vma_complete(&vma_prep, vmi, mm);
        vma_iter_free(vmi);
        validate_mm(mm);