s390/mm: uninline pmdp_xxx functions from pgtable.h

The pmdp_xxx function are smaller than their ptep_xxx counterparts
but to keep things symmetrical unline them as well.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index d102c4e..572001c 100644 (file)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -520,15 +520,6 @@ static inline int pmd_bad(pmd_t pmd)
        return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
 }
 
-#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
-extern int pmdp_set_access_flags(struct vm_area_struct *vma,
-                                unsigned long address, pmd_t *pmdp,
-                                pmd_t entry, int dirty);
-
-#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
-extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
-                                 unsigned long address, pmd_t *pmdp);
-
 #define __HAVE_ARCH_PMD_WRITE
 static inline int pmd_write(pmd_t pmd)
 {
@@ -1203,54 +1194,51 @@ static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp)
                : "cc" );
 }
 
-static inline void pmdp_flush_direct(struct mm_struct *mm,
-                                    unsigned long address, pmd_t *pmdp)
-{
-       int active, count;
+pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
+pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
 
-       if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
-               return;
-       if (!MACHINE_HAS_IDTE) {
-               __pmdp_csp(pmdp);
-               return;
-       }
-       active = (mm == current->active_mm) ? 1 : 0;
-       count = atomic_add_return(0x10000, &mm->context.attach_count);
-       if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
-           cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
-               __pmdp_idte_local(address, pmdp);
-       else
-               __pmdp_idte(address, pmdp);
-       atomic_sub(0x10000, &mm->context.attach_count);
-}
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+
+#define __HAVE_ARCH_PGTABLE_DEPOSIT
+void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
+                               pgtable_t pgtable);
+
+#define __HAVE_ARCH_PGTABLE_WITHDRAW
+pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
 
-static inline void pmdp_flush_lazy(struct mm_struct *mm,
-                                  unsigned long address, pmd_t *pmdp)
+#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
+static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
+                                       unsigned long addr, pmd_t *pmdp,
+                                       pmd_t entry, int dirty)
 {
-       int active, count;
+       VM_BUG_ON(addr & ~HPAGE_MASK);
 
-       if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
-               return;
-       active = (mm == current->active_mm) ? 1 : 0;
-       count = atomic_add_return(0x10000, &mm->context.attach_count);
-       if ((count & 0xffff) <= active) {
-               pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
-               mm->context.flush_mm = 1;
-       } else if (MACHINE_HAS_IDTE)
-               __pmdp_idte(address, pmdp);
-       else
-               __pmdp_csp(pmdp);
-       atomic_sub(0x10000, &mm->context.attach_count);
+       entry = pmd_mkyoung(entry);
+       if (dirty)
+               entry = pmd_mkdirty(entry);
+       if (pmd_val(*pmdp) == pmd_val(entry))
+               return 0;
+       pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);
+       return 1;
 }
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
+static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
+                                           unsigned long addr, pmd_t *pmdp)
+{
+       pmd_t pmd = *pmdp;
 
-#define __HAVE_ARCH_PGTABLE_DEPOSIT
-extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
-                                      pgtable_t pgtable);
+       pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));
+       return pmd_young(pmd);
+}
 
-#define __HAVE_ARCH_PGTABLE_WITHDRAW
-extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
+#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
+static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
+                                        unsigned long addr, pmd_t *pmdp)
+{
+       VM_BUG_ON(addr & ~HPAGE_MASK);
+       return pmdp_test_and_clear_young(vma, addr, pmdp);
+}
 
 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
                              pmd_t *pmdp, pmd_t entry)
@@ -1266,66 +1254,48 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd)
        return pmd;
 }
 
-#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
-static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
-                                           unsigned long address, pmd_t *pmdp)
-{
-       pmd_t pmd;
-
-       pmd = *pmdp;
-       pmdp_flush_direct(vma->vm_mm, address, pmdp);
-       *pmdp = pmd_mkold(pmd);
-       return pmd_young(pmd);
-}
-
 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
-                                           unsigned long address, pmd_t *pmdp)
+                                           unsigned long addr, pmd_t *pmdp)
 {
-       pmd_t pmd = *pmdp;
-
-       pmdp_flush_direct(mm, address, pmdp);
-       pmd_clear(pmdp);
-       return pmd;
+       return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
 }
 
 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
 static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm,
-                                                unsigned long address,
+                                                unsigned long addr,
                                                 pmd_t *pmdp, int full)
 {
-       pmd_t pmd = *pmdp;
-
-       if (!full)
-               pmdp_flush_lazy(mm, address, pmdp);
-       pmd_clear(pmdp);
-       return pmd;
+       if (full) {
+               pmd_t pmd = *pmdp;
+               *pmdp = __pmd(_SEGMENT_ENTRY_INVALID);
+               return pmd;
+       }
+       return pmdp_xchg_lazy(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
 }
 
 #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
 static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
-                                         unsigned long address, pmd_t *pmdp)
+                                         unsigned long addr, pmd_t *pmdp)
 {
-       return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
+       return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
 }
 
 #define __HAVE_ARCH_PMDP_INVALIDATE
 static inline void pmdp_invalidate(struct vm_area_struct *vma,
-                                  unsigned long address, pmd_t *pmdp)
+                                  unsigned long addr, pmd_t *pmdp)
 {
-       pmdp_flush_direct(vma->vm_mm, address, pmdp);
+       pmdp_xchg_direct(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
 }
 
 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
-                                     unsigned long address, pmd_t *pmdp)
+                                     unsigned long addr, pmd_t *pmdp)
 {
        pmd_t pmd = *pmdp;
 
-       if (pmd_write(pmd)) {
-               pmdp_flush_direct(mm, address, pmdp);
-               set_pmd_at(mm, address, pmdp, pmd_wrprotect(pmd));
-       }
+       if (pmd_write(pmd))
+               pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd));
 }
 
 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,

diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index f81096b..1b5e898 100644 (file)
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -105,11 +105,10 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
                              unsigned long addr, pte_t *ptep)
 {
        pmd_t *pmdp = (pmd_t *) ptep;
-       pte_t pte = huge_ptep_get(ptep);
+       pmd_t old;
 
-       pmdp_flush_direct(mm, addr, pmdp);
-       pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
-       return pte;
+       old = pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+       return __pmd_to_pte(old);
 }
 
 pte_t *huge_pte_alloc(struct mm_struct *mm,

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 30033aa..e241262 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -1418,6 +1418,74 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 }
 EXPORT_SYMBOL(ptep_modify_prot_commit);
 
+static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
+                                     unsigned long addr, pmd_t *pmdp)
+{
+       int active, count;
+       pmd_t old;
+
+       old = *pmdp;
+       if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
+               return old;
+       if (!MACHINE_HAS_IDTE) {
+               __pmdp_csp(pmdp);
+               return old;
+       }
+       active = (mm == current->active_mm) ? 1 : 0;
+       count = atomic_add_return(0x10000, &mm->context.attach_count);
+       if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
+           cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
+               __pmdp_idte_local(addr, pmdp);
+       else
+               __pmdp_idte(addr, pmdp);
+       atomic_sub(0x10000, &mm->context.attach_count);
+       return old;
+}
+
+static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
+                                   unsigned long addr, pmd_t *pmdp)
+{
+       int active, count;
+       pmd_t old;
+
+       old = *pmdp;
+       if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
+               return old;
+       active = (mm == current->active_mm) ? 1 : 0;
+       count = atomic_add_return(0x10000, &mm->context.attach_count);
+       if ((count & 0xffff) <= active) {
+               pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
+               mm->context.flush_mm = 1;
+       } else if (MACHINE_HAS_IDTE)
+               __pmdp_idte(addr, pmdp);
+       else
+               __pmdp_csp(pmdp);
+       atomic_sub(0x10000, &mm->context.attach_count);
+       return old;
+}
+
+pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
+                      pmd_t *pmdp, pmd_t new)
+{
+       pmd_t old;
+
+       old = pmdp_flush_direct(mm, addr, pmdp);
+       *pmdp = new;
+       return old;
+}
+EXPORT_SYMBOL(pmdp_xchg_direct);
+
+pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
+                    pmd_t *pmdp, pmd_t new)
+{
+       pmd_t old;
+
+       old = pmdp_flush_lazy(mm, addr, pmdp);
+       *pmdp = new;
+       return old;
+}
+EXPORT_SYMBOL(pmdp_xchg_lazy);
+
 /*
  * switch on pgstes for its userspace process (for kvm)
  */
@@ -1525,31 +1593,6 @@ void s390_reset_cmma(struct mm_struct *mm)
 EXPORT_SYMBOL_GPL(s390_reset_cmma);
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
-                          pmd_t *pmdp)
-{
-       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
-       /* No need to flush TLB
-        * On s390 reference bits are in storage key and never in TLB */
-       return pmdp_test_and_clear_young(vma, address, pmdp);
-}
-
-int pmdp_set_access_flags(struct vm_area_struct *vma,
-                         unsigned long address, pmd_t *pmdp,
-                         pmd_t entry, int dirty)
-{
-       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
-
-       entry = pmd_mkyoung(entry);
-       if (dirty)
-               entry = pmd_mkdirty(entry);
-       if (pmd_same(*pmdp, entry))
-               return 0;
-       pmdp_invalidate(vma, address, pmdp);
-       set_pmd_at(vma->vm_mm, address, pmdp, entry);
-       return 1;
-}
-
 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
                                pgtable_t pgtable)
 {