mm/memory.c: Update local TLB if PTE entry exists

If two threads concurrently fault at the same page, the thread that
won the race updates the PTE and its local TLB. For now, the other
thread gives up, simply does nothing, and continues.

It could happen that this second thread triggers another fault, whereby
it only updates its local TLB while handling the fault. Instead of
triggering another fault, let's directly update the local TLB of the
second thread. Function update_mmu_tlb is used here to update local
TLB on the second thread, and it is defined as empty on other arches.

Signed-off-by: Bibo Mao <maobibo@loongson.cn>
Acked-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Thomas Bogendoerfer <tsbogend@alpha.franken.de>

diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
index f8f48fc361d5b48df59bfbc471b637ae623cae49..6f40612b3152ed8dd63bd85aa68dfa9ce48f7e0a 100644 (file)
--- a/arch/mips/include/asm/pgtable.h
+++ b/arch/mips/include/asm/pgtable.h
@@ -483,6 +483,26 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
 {
 }
 
+#define __HAVE_ARCH_PTE_SAME
+static inline int pte_same(pte_t pte_a, pte_t pte_b)
+{
+       return pte_val(pte_a) == pte_val(pte_b);
+}
+
+#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
+static inline int ptep_set_access_flags(struct vm_area_struct *vma,
+                                       unsigned long address, pte_t *ptep,
+                                       pte_t entry, int dirty)
+{
+       if (!pte_same(*ptep, entry))
+               set_pte_at(vma->vm_mm, address, ptep, entry);
+       /*
+        * update_mmu_cache will unconditionally execute, handling both
+        * the case that the PTE changed and the spurious fault case.
+        */
+       return true;
+}
+
 /*
  * Conversion functions: convert a page and protection to a page entry,
  * and a page entry and page directory to the page they refer to.
@@ -526,6 +546,9 @@ static inline void update_mmu_cache(struct vm_area_struct *vma,
        __update_tlb(vma, address, pte);
 }
 
+#define        __HAVE_ARCH_UPDATE_MMU_TLB
+#define update_mmu_tlb update_mmu_cache
+
 static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
        unsigned long address, pmd_t *pmdp)
 {

diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index 329b8c8ca703b670c149e7eaf840682c523f9d7c..fa5c73ff82268b083c8983522f07762a06fbda73 100644 (file)
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -188,6 +188,23 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
 }
 #endif
 
+
+/*
+ * If two threads concurrently fault at the same page, the thread that
+ * won the race updates the PTE and its local TLB/Cache. The other thread
+ * gives up, simply does nothing, and continues; on architectures where
+ * software can update TLB,  local TLB can be updated here to avoid next page
+ * fault. This function updates TLB only, do nothing with cache or others.
+ * It is the difference with function update_mmu_cache.
+ */
+#ifndef __HAVE_ARCH_UPDATE_MMU_TLB
+static inline void update_mmu_tlb(struct vm_area_struct *vma,
+                               unsigned long address, pte_t *ptep)
+{
+}
+#define __HAVE_ARCH_UPDATE_MMU_TLB
+#endif
+
 /*
  * Some architectures may be able to avoid expensive synchronization
  * primitives when modifications are made to PTE's which are already

diff --git a/mm/memory.c b/mm/memory.c
index f703fe8c83460202354ca1c8cc693eb99b682c2e..8bb31c4d94d0df24e95362ba94ae047bdb8b36b6 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2436,10 +2436,9 @@ static inline bool cow_user_page(struct page *dst, struct page *src,
                if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
                        /*
                         * Other thread has already handled the fault
-                        * and we don't need to do anything. If it's
-                        * not the case, the fault will be triggered
-                        * again on the same address.
+                        * and update local tlb only
                         */
+                       update_mmu_tlb(vma, addr, vmf->pte);
                        ret = false;
                        goto pte_unlock;
                }
@@ -2463,7 +2462,8 @@ static inline bool cow_user_page(struct page *dst, struct page *src,
                vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
                locked = true;
                if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
-                       /* The PTE changed under us. Retry page fault. */
+                       /* The PTE changed under us, update local tlb */
+                       update_mmu_tlb(vma, addr, vmf->pte);
                        ret = false;
                        goto pte_unlock;
                }
@@ -2752,6 +2752,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
                new_page = old_page;
                page_copied = 1;
        } else {
+               update_mmu_tlb(vma, vmf->address, vmf->pte);
                mem_cgroup_cancel_charge(new_page, memcg, false);
        }
 
@@ -2812,6 +2813,7 @@ vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
         * pte_offset_map_lock.
         */
        if (!pte_same(*vmf->pte, vmf->orig_pte)) {
+               update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                return VM_FAULT_NOPAGE;
        }
@@ -2936,6 +2938,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
                        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                        vmf->address, &vmf->ptl);
                        if (!pte_same(*vmf->pte, vmf->orig_pte)) {
+                               update_mmu_tlb(vma, vmf->address, vmf->pte);
                                unlock_page(vmf->page);
                                pte_unmap_unlock(vmf->pte, vmf->ptl);
                                put_page(vmf->page);
@@ -3341,8 +3344,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
                                                vma->vm_page_prot));
                vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                vmf->address, &vmf->ptl);
-               if (!pte_none(*vmf->pte))
+               if (!pte_none(*vmf->pte)) {
+                       update_mmu_tlb(vma, vmf->address, vmf->pte);
                        goto unlock;
+               }
                ret = check_stable_address_space(vma->vm_mm);
                if (ret)
                        goto unlock;
@@ -3378,8 +3383,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 
        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
                        &vmf->ptl);
-       if (!pte_none(*vmf->pte))
+       if (!pte_none(*vmf->pte)) {
+               update_mmu_cache(vma, vmf->address, vmf->pte);
                goto release;
+       }
 
        ret = check_stable_address_space(vma->vm_mm);
        if (ret)
@@ -3646,8 +3653,10 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
        }
 
        /* Re-check under ptl */
-       if (unlikely(!pte_none(*vmf->pte)))
+       if (unlikely(!pte_none(*vmf->pte))) {
+               update_mmu_tlb(vma, vmf->address, vmf->pte);
                return VM_FAULT_NOPAGE;
+       }
 
        flush_icache_page(vma, page);
        entry = mk_pte(page, vma->vm_page_prot);
@@ -4224,8 +4233,10 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
        vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
        spin_lock(vmf->ptl);
        entry = vmf->orig_pte;
-       if (unlikely(!pte_same(*vmf->pte, entry)))
+       if (unlikely(!pte_same(*vmf->pte, entry))) {
+               update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
                goto unlock;
+       }
        if (vmf->flags & FAULT_FLAG_WRITE) {
                if (!pte_write(entry))
                        return do_wp_page(vmf);