ksm: distribute remove_mm_from_lists

Do some housekeeping in ksm.c, to help make the next patch easier
to understand: remove the function remove_mm_from_lists, distributing
its code to its callsites scan_get_next_rmap_item and __ksm_exit.

That turns out to be a win in scan_get_next_rmap_item: move its
remove_trailing_rmap_items and cursor advancement up, and it becomes
simpler than before.  __ksm_exit becomes messier, but will change
again; and moving its remove_trailing_rmap_items up lets us strengthen
the unstable tree item's age condition in remove_rmap_item_from_tree.

Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Acked-by: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/ksm.c b/mm/ksm.c
index d9e3cfc..7e4d255 100644 (file)
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -444,14 +444,9 @@ static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
                 * But __ksm_exit has to be careful: do the rb_erase
                 * if it's interrupting a scan, and this rmap_item was
                 * inserted by this scan rather than left from before.
-                *
-                * Because of the case in which remove_mm_from_lists
-                * increments seqnr before removing rmaps, unstable_nr
-                * may even be 2 behind seqnr, but should never be
-                * further behind.  Yes, I did have trouble with this!
                 */
                age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
-               BUG_ON(age > 2);
+               BUG_ON(age > 1);
                if (!age)
                        rb_erase(&rmap_item->node, &root_unstable_tree);
                ksm_pages_unshared--;
@@ -546,37 +541,6 @@ out:
        return err;
 }
 
-static void remove_mm_from_lists(struct mm_struct *mm)
-{
-       struct mm_slot *mm_slot;
-
-       spin_lock(&ksm_mmlist_lock);
-       mm_slot = get_mm_slot(mm);
-
-       /*
-        * This mm_slot is always at the scanning cursor when we're
-        * called from scan_get_next_rmap_item; but it's a special
-        * case when we're called from __ksm_exit.
-        */
-       if (ksm_scan.mm_slot == mm_slot) {
-               ksm_scan.mm_slot = list_entry(
-                       mm_slot->mm_list.next, struct mm_slot, mm_list);
-               ksm_scan.address = 0;
-               ksm_scan.rmap_item = list_entry(
-                       &ksm_scan.mm_slot->rmap_list, struct rmap_item, link);
-               if (ksm_scan.mm_slot == &ksm_mm_head)
-                       ksm_scan.seqnr++;
-       }
-
-       hlist_del(&mm_slot->link);
-       list_del(&mm_slot->mm_list);
-       spin_unlock(&ksm_mmlist_lock);
-
-       remove_trailing_rmap_items(mm_slot, mm_slot->rmap_list.next);
-       free_mm_slot(mm_slot);
-       clear_bit(MMF_VM_MERGEABLE, &mm->flags);
-}
-
 static u32 calc_checksum(struct page *page)
 {
        u32 checksum;
@@ -1241,33 +1205,31 @@ next_mm:
                }
        }
 
-       if (!ksm_scan.address) {
-               /*
-                * We've completed a full scan of all vmas, holding mmap_sem
-                * throughout, and found no VM_MERGEABLE: so do the same as
-                * __ksm_exit does to remove this mm from all our lists now.
-                */
-               remove_mm_from_lists(mm);
-               up_read(&mm->mmap_sem);
-               slot = ksm_scan.mm_slot;
-               if (slot != &ksm_mm_head)
-                       goto next_mm;
-               return NULL;
-       }
-
        /*
         * Nuke all the rmap_items that are above this current rmap:
         * because there were no VM_MERGEABLE vmas with such addresses.
         */
        remove_trailing_rmap_items(slot, ksm_scan.rmap_item->link.next);
-       up_read(&mm->mmap_sem);
 
        spin_lock(&ksm_mmlist_lock);
-       slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
-       ksm_scan.mm_slot = slot;
+       ksm_scan.mm_slot = list_entry(slot->mm_list.next,
+                                               struct mm_slot, mm_list);
+       if (ksm_scan.address == 0) {
+               /*
+                * We've completed a full scan of all vmas, holding mmap_sem
+                * throughout, and found no VM_MERGEABLE: so do the same as
+                * __ksm_exit does to remove this mm from all our lists now.
+                */
+               hlist_del(&slot->link);
+               list_del(&slot->mm_list);
+               free_mm_slot(slot);
+               clear_bit(MMF_VM_MERGEABLE, &mm->flags);
+       }
        spin_unlock(&ksm_mmlist_lock);
+       up_read(&mm->mmap_sem);
 
        /* Repeat until we've completed scanning the whole list */
+       slot = ksm_scan.mm_slot;
        if (slot != &ksm_mm_head)
                goto next_mm;
 
@@ -1408,13 +1370,38 @@ int __ksm_enter(struct mm_struct *mm)
 
 void __ksm_exit(struct mm_struct *mm)
 {
+       struct mm_slot *mm_slot;
+
        /*
         * This process is exiting: doesn't hold and doesn't need mmap_sem;
         * but we do need to exclude ksmd and other exiters while we modify
         * the various lists and trees.
         */
        mutex_lock(&ksm_thread_mutex);
-       remove_mm_from_lists(mm);
+       spin_lock(&ksm_mmlist_lock);
+       mm_slot = get_mm_slot(mm);
+       if (!list_empty(&mm_slot->rmap_list)) {
+               spin_unlock(&ksm_mmlist_lock);
+               remove_trailing_rmap_items(mm_slot, mm_slot->rmap_list.next);
+               spin_lock(&ksm_mmlist_lock);
+       }
+
+       if (ksm_scan.mm_slot == mm_slot) {
+               ksm_scan.mm_slot = list_entry(
+                       mm_slot->mm_list.next, struct mm_slot, mm_list);
+               ksm_scan.address = 0;
+               ksm_scan.rmap_item = list_entry(
+                       &ksm_scan.mm_slot->rmap_list, struct rmap_item, link);
+               if (ksm_scan.mm_slot == &ksm_mm_head)
+                       ksm_scan.seqnr++;
+       }
+
+       hlist_del(&mm_slot->link);
+       list_del(&mm_slot->mm_list);
+       spin_unlock(&ksm_mmlist_lock);
+
+       free_mm_slot(mm_slot);
+       clear_bit(MMF_VM_MERGEABLE, &mm->flags);
        mutex_unlock(&ksm_thread_mutex);
 }