parisc: Use per-pagetable spinlock

PA-RISC uses a global spinlock to protect pagetable updates in the TLB
fault handlers. When multiple cores are taking TLB faults simultaneously,
the cache line containing the spinlock becomes a bottleneck.

This patch embeds the spinlock in the top level page directory, so that
every process has its own lock. It improves performance by 30% when
doing parallel compilations.

At least on the N class systems, only one PxTLB inter processor
broadcast can be active at any one time on the Merced bus. If a Merced
bus is found, this patch serializes the TLB flushes with the
pa_tlb_flush_lock spinlock.

v1: Initial patch by Mikulas
v2: Added Merced detection by Helge
v3: Revised TLB serialization by Dave & Helge

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: John David Anglin <dave.anglin@bell.net>
Signed-off-by: Helge Deller <deller@gmx.de>

diff --git a/arch/parisc/include/asm/hardware.h b/arch/parisc/include/asm/hardware.h
index d6e1ed1..9d3d773 100644 (file)
--- a/arch/parisc/include/asm/hardware.h
+++ b/arch/parisc/include/asm/hardware.h
@@ -120,7 +120,7 @@ extern void get_pci_node_path(struct pci_dev *dev, struct hardware_path *path);
 extern void init_parisc_bus(void);
 extern struct device *hwpath_to_device(struct hardware_path *modpath);
 extern void device_to_hwpath(struct device *dev, struct hardware_path *path);
-
+extern int machine_has_merced_bus(void);
 
 /* inventory.c: */
 extern void do_memory_inventory(void);

diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h
index d05c678..ea75cc9 100644 (file)
--- a/arch/parisc/include/asm/pgalloc.h
+++ b/arch/parisc/include/asm/pgalloc.h
@@ -41,6 +41,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
                __pgd_val_set(*pgd, PxD_FLAG_ATTACHED);
 #endif
        }
+       spin_lock_init(pgd_spinlock(actual_pgd));
        return actual_pgd;
 }
 

diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h
index c7bb74e..a39b079 100644 (file)
--- a/arch/parisc/include/asm/pgtable.h
+++ b/arch/parisc/include/asm/pgtable.h
@@ -17,7 +17,7 @@
 #include <asm/processor.h>
 #include <asm/cache.h>
 
-extern spinlock_t pa_tlb_lock;
+static inline spinlock_t *pgd_spinlock(pgd_t *);
 
 /*
  * kern_addr_valid(ADDR) tests if ADDR is pointing to valid kernel
@@ -34,16 +34,46 @@ extern spinlock_t pa_tlb_lock;
  */
 #define kern_addr_valid(addr)  (1)
 
-/* Purge data and instruction TLB entries.  Must be called holding
- * the pa_tlb_lock.  The TLB purge instructions are slow on SMP
- * machines since the purge must be broadcast to all CPUs.
+/* This is for the serialization of PxTLB broadcasts. At least on the N class
+ * systems, only one PxTLB inter processor broadcast can be active at any one
+ * time on the Merced bus.
+
+ * PTE updates are protected by locks in the PMD.
+ */
+extern spinlock_t pa_tlb_flush_lock;
+extern spinlock_t pa_swapper_pg_lock;
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+extern int pa_serialize_tlb_flushes;
+#else
+#define pa_serialize_tlb_flushes        (0)
+#endif
+
+#define purge_tlb_start(flags)  do { \
+       if (pa_serialize_tlb_flushes)   \
+               spin_lock_irqsave(&pa_tlb_flush_lock, flags); \
+       else \
+               local_irq_save(flags);  \
+       } while (0)
+#define purge_tlb_end(flags)   do { \
+       if (pa_serialize_tlb_flushes)   \
+               spin_unlock_irqrestore(&pa_tlb_flush_lock, flags); \
+       else \
+               local_irq_restore(flags); \
+       } while (0)
+
+/* Purge data and instruction TLB entries. The TLB purge instructions
+ * are slow on SMP machines since the purge must be broadcast to all CPUs.
  */
 
 static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
 {
+       unsigned long flags;
+
+       purge_tlb_start(flags);
        mtsp(mm->context, 1);
        pdtlb(addr);
        pitlb(addr);
+       purge_tlb_end(flags);
 }
 
 /* Certain architectures need to do special things when PTEs
@@ -59,11 +89,11 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
        do {                                                    \
                pte_t old_pte;                                  \
                unsigned long flags;                            \
-               spin_lock_irqsave(&pa_tlb_lock, flags);         \
+               spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);\
                old_pte = *ptep;                                \
                set_pte(ptep, pteval);                          \
                purge_tlb_entries(mm, addr);                    \
-               spin_unlock_irqrestore(&pa_tlb_lock, flags);    \
+               spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);\
        } while (0)
 
 #endif /* !__ASSEMBLY__ */
@@ -88,10 +118,10 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
 #if CONFIG_PGTABLE_LEVELS == 3
 #define PGD_ORDER      1 /* Number of pages per pgd */
 #define PMD_ORDER      1 /* Number of pages per pmd */
-#define PGD_ALLOC_ORDER        2 /* first pgd contains pmd */
+#define PGD_ALLOC_ORDER        (2 + 1) /* first pgd contains pmd */
 #else
 #define PGD_ORDER      1 /* Number of pages per pgd */
-#define PGD_ALLOC_ORDER        PGD_ORDER
+#define PGD_ALLOC_ORDER        (PGD_ORDER + 1)
 #endif
 
 /* Definitions for 3rd level (we use PLD here for Page Lower directory
@@ -459,6 +489,15 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *);
 #define __pte_to_swp_entry(pte)                ((swp_entry_t) { pte_val(pte) })
 #define __swp_entry_to_pte(x)          ((pte_t) { (x).val })
 
+
+static inline spinlock_t *pgd_spinlock(pgd_t *pgd)
+{
+       if (unlikely(pgd == swapper_pg_dir))
+               return &pa_swapper_pg_lock;
+       return (spinlock_t *)((char *)pgd + (PAGE_SIZE << (PGD_ALLOC_ORDER - 1)));
+}
+
+
 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
 {
        pte_t pte;
@@ -467,15 +506,15 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned
        if (!pte_young(*ptep))
                return 0;
 
-       spin_lock_irqsave(&pa_tlb_lock, flags);
+       spin_lock_irqsave(pgd_spinlock(vma->vm_mm->pgd), flags);
        pte = *ptep;
        if (!pte_young(pte)) {
-               spin_unlock_irqrestore(&pa_tlb_lock, flags);
+               spin_unlock_irqrestore(pgd_spinlock(vma->vm_mm->pgd), flags);
                return 0;
        }
        set_pte(ptep, pte_mkold(pte));
        purge_tlb_entries(vma->vm_mm, addr);
-       spin_unlock_irqrestore(&pa_tlb_lock, flags);
+       spin_unlock_irqrestore(pgd_spinlock(vma->vm_mm->pgd), flags);
        return 1;
 }
 
@@ -485,11 +524,11 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
        pte_t old_pte;
        unsigned long flags;
 
-       spin_lock_irqsave(&pa_tlb_lock, flags);
+       spin_lock_irqsave(pgd_spinlock(mm->pgd), flags);
        old_pte = *ptep;
        set_pte(ptep, __pte(0));
        purge_tlb_entries(mm, addr);
-       spin_unlock_irqrestore(&pa_tlb_lock, flags);
+       spin_unlock_irqrestore(pgd_spinlock(mm->pgd), flags);
 
        return old_pte;
 }
@@ -497,10 +536,10 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
        unsigned long flags;
-       spin_lock_irqsave(&pa_tlb_lock, flags);
+       spin_lock_irqsave(pgd_spinlock(mm->pgd), flags);
        set_pte(ptep, pte_wrprotect(*ptep));
        purge_tlb_entries(mm, addr);
-       spin_unlock_irqrestore(&pa_tlb_lock, flags);
+       spin_unlock_irqrestore(pgd_spinlock(mm->pgd), flags);
 }
 
 #define pte_same(A,B)  (pte_val(A) == pte_val(B))

diff --git a/arch/parisc/include/asm/tlbflush.h b/arch/parisc/include/asm/tlbflush.h
index 6804374..c5ded01 100644 (file)
--- a/arch/parisc/include/asm/tlbflush.h
+++ b/arch/parisc/include/asm/tlbflush.h
@@ -8,21 +8,6 @@
 #include <linux/sched.h>
 #include <asm/mmu_context.h>
 
-
-/* This is for the serialisation of PxTLB broadcasts.  At least on the
- * N class systems, only one PxTLB inter processor broadcast can be
- * active at any one time on the Merced bus.  This tlb purge
- * synchronisation is fairly lightweight and harmless so we activate
- * it on all systems not just the N class.
-
- * It is also used to ensure PTE updates are atomic and consistent
- * with the TLB.
- */
-extern spinlock_t pa_tlb_lock;
-
-#define purge_tlb_start(flags) spin_lock_irqsave(&pa_tlb_lock, flags)
-#define purge_tlb_end(flags)   spin_unlock_irqrestore(&pa_tlb_lock, flags)
-
 extern void flush_tlb_all(void);
 extern void flush_tlb_all_local(void *);
 
@@ -79,13 +64,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
 static inline void flush_tlb_page(struct vm_area_struct *vma,
        unsigned long addr)
 {
-       unsigned long flags, sid;
-
-       sid = vma->vm_mm->context;
-       purge_tlb_start(flags);
-       mtsp(sid, 1);
-       pdtlb(addr);
-       pitlb(addr);
-       purge_tlb_end(flags);
+       purge_tlb_entries(vma->vm_mm, addr);
 }
 #endif

diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index 804880e..0338561 100644 (file)
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -40,12 +40,19 @@ void purge_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr);
 void flush_icache_page_asm(unsigned long phys_addr, unsigned long vaddr);
 
 
-/* On some machines (e.g. ones with the Merced bus), there can be
+/* On some machines (i.e., ones with the Merced bus), there can be
  * only a single PxTLB broadcast at a time; this must be guaranteed
- * by software.  We put a spinlock around all TLB flushes  to
- * ensure this.
+ * by software. We need a spinlock around all TLB flushes to ensure
+ * this.
  */
-DEFINE_SPINLOCK(pa_tlb_lock);
+DEFINE_SPINLOCK(pa_tlb_flush_lock);
+
+/* Swapper page setup lock. */
+DEFINE_SPINLOCK(pa_swapper_pg_lock);
+
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+int pa_serialize_tlb_flushes __read_mostly;
+#endif
 
 struct pdc_cache_info cache_info __read_mostly;
 #ifndef CONFIG_PA20

diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c
index 5eb979d..15e7b3b 100644 (file)
--- a/arch/parisc/kernel/drivers.c
+++ b/arch/parisc/kernel/drivers.c
@@ -38,6 +38,7 @@
 #include <asm/io.h>
 #include <asm/pdc.h>
 #include <asm/parisc-device.h>
+#include <asm/ropes.h>
 
 /* See comments in include/asm-parisc/pci.h */
 const struct dma_map_ops *hppa_dma_ops __read_mostly;
@@ -257,6 +258,30 @@ static struct parisc_device *find_device_by_addr(unsigned long hpa)
        return ret ? d.dev : NULL;
 }
 
+static int __init is_IKE_device(struct device *dev, void *data)
+{
+       struct parisc_device *pdev = to_parisc_device(dev);
+
+       if (!check_dev(dev))
+               return 0;
+       if (pdev->id.hw_type != HPHW_BCPORT)
+               return 0;
+       if (IS_IKE(pdev) ||
+               (pdev->id.hversion == REO_MERCED_PORT) ||
+               (pdev->id.hversion == REOG_MERCED_PORT)) {
+                       return 1;
+       }
+       return 0;
+}
+
+int __init machine_has_merced_bus(void)
+{
+       int ret;
+
+       ret = for_each_padev(is_IKE_device, NULL);
+       return ret ? 1 : 0;
+}
+
 /**
  * find_pa_parent_type - Find a parent of a specific type
  * @dev: The device to start searching from

diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
index 5796524..a1fc045 100644 (file)
--- a/arch/parisc/kernel/entry.S
+++ b/arch/parisc/kernel/entry.S
@@ -50,12 +50,8 @@
 
        .import         pa_tlb_lock,data
        .macro  load_pa_tlb_lock reg
-#if __PA_LDCW_ALIGNMENT > 4
-       load32  PA(pa_tlb_lock) + __PA_LDCW_ALIGNMENT-1, \reg
-       depi    0,31,__PA_LDCW_ALIGN_ORDER, \reg
-#else
-       load32  PA(pa_tlb_lock), \reg
-#endif
+       mfctl           %cr25,\reg
+       addil           L%(PAGE_SIZE << (PGD_ALLOC_ORDER - 1)),\reg
        .endm
 
        /* space_to_prot macro creates a prot id from a space id */

diff --git a/arch/parisc/kernel/inventory.c b/arch/parisc/kernel/inventory.c
index 35d05fd..6f2d611 100644 (file)
--- a/arch/parisc/kernel/inventory.c
+++ b/arch/parisc/kernel/inventory.c
@@ -31,6 +31,7 @@
 #include <asm/processor.h>
 #include <asm/page.h>
 #include <asm/parisc-device.h>
+#include <asm/tlbflush.h>
 
 /*
 ** Debug options
@@ -638,4 +639,10 @@ void __init do_device_inventory(void)
        }
        printk(KERN_INFO "Found devices:\n");
        print_parisc_devices();
+
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+       pa_serialize_tlb_flushes = machine_has_merced_bus();
+       if (pa_serialize_tlb_flushes)
+               pr_info("Merced bus found: Enable PxTLB serialization.\n");
+#endif
 }

diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c
index d908058..e05cb2a 100644 (file)
--- a/arch/parisc/kernel/setup.c
+++ b/arch/parisc/kernel/setup.c
@@ -343,6 +343,12 @@ static int __init parisc_init(void)
                        boot_cpu_data.cpu_hz / 1000000,
                        boot_cpu_data.cpu_hz % 1000000  );
 
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+       /* Don't serialize TLB flushes if we run on one CPU only. */
+       if (num_online_cpus() == 1)
+               pa_serialize_tlb_flushes = 0;
+#endif
+
        apply_alternatives_all();
        parisc_setup_cache_timing();