powerpc/e6500: hw tablewalk: make sure we invalidate and write to the same tlb entry

In order to workaround Erratum A-008139, we have to invalidate the
tlb entry with tlbilx before overwriting. Due to the performance
consideration, we don't add any memory barrier when acquire/release
the tcd lock. This means the two load instructions for esel_next do
have the possibility to return different value. This is definitely
not acceptable due to the Erratum A-008139. We have two options to
fix this issue:
  a) Add memory barrier when acquire/release tcd lock to order the
     load/store to esel_next.
  b) Just make sure to invalidate and write to the same tlb entry and
     tolerate the race that we may get the wrong value and overwrite
     the tlb entry just updated by the other thread.

We observe better performance using option b. So reserve an additional
register to save the value of the esel_next.

Signed-off-by: Kevin Hao <haokexin@gmail.com>
Signed-off-by: Scott Wood <scottwood@freescale.com>

diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h
index 344fc43..a703452 100644 (file)
--- a/arch/powerpc/include/asm/exception-64e.h
+++ b/arch/powerpc/include/asm/exception-64e.h
@@ -69,13 +69,14 @@
 #define EX_TLB_ESR     ( 9 * 8) /* Level 0 and 2 only */
 #define EX_TLB_SRR0    (10 * 8)
 #define EX_TLB_SRR1    (11 * 8)
+#define EX_TLB_R7      (12 * 8)
 #ifdef CONFIG_BOOK3E_MMU_TLB_STATS
-#define EX_TLB_R8      (12 * 8)
-#define EX_TLB_R9      (13 * 8)
-#define EX_TLB_LR      (14 * 8)
-#define EX_TLB_SIZE    (15 * 8)
+#define EX_TLB_R8      (13 * 8)
+#define EX_TLB_R9      (14 * 8)
+#define EX_TLB_LR      (15 * 8)
+#define EX_TLB_SIZE    (16 * 8)
 #else
-#define EX_TLB_SIZE    (12 * 8)
+#define EX_TLB_SIZE    (13 * 8)
 #endif
 
 #define        START_EXCEPTION(label)                                          \

diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S
index e418558..29d6987 100644 (file)
--- a/arch/powerpc/mm/tlb_low_64e.S
+++ b/arch/powerpc/mm/tlb_low_64e.S
@@ -68,11 +68,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
        ld      r14,PACAPGD(r13)
        std     r15,EX_TLB_R15(r12)
        std     r10,EX_TLB_CR(r12)
+#ifdef CONFIG_PPC_FSL_BOOK3E
+       std     r7,EX_TLB_R7(r12)
+#endif
        TLB_MISS_PROLOG_STATS
 .endm
 
 .macro tlb_epilog_bolted
        ld      r14,EX_TLB_CR(r12)
+#ifdef CONFIG_PPC_FSL_BOOK3E
+       ld      r7,EX_TLB_R7(r12)
+#endif
        ld      r10,EX_TLB_R10(r12)
        ld      r11,EX_TLB_R11(r12)
        ld      r13,EX_TLB_R13(r12)
@@ -297,6 +303,7 @@ itlb_miss_fault_bolted:
  * r13 = PACA
  * r11 = tlb_per_core ptr
  * r10 = crap (free to use)
+ * r7  = esel_next
  */
 tlb_miss_common_e6500:
        crmove  cr2*4+2,cr0*4+2         /* cr2.eq != 0 if kernel address */
@@ -325,7 +332,11 @@ BEGIN_FTR_SECTION          /* CPU_FTR_SMT */
        bne     10b
        b       1b
        .previous
+END_FTR_SECTION_IFSET(CPU_FTR_SMT)
+
+       lbz     r7,TCD_ESEL_NEXT(r11)
 
+BEGIN_FTR_SECTION              /* CPU_FTR_SMT */
        /*
         * Erratum A-008139 says that we can't use tlbwe to change
         * an indirect entry in any way (including replacing or
@@ -334,8 +345,7 @@ BEGIN_FTR_SECTION           /* CPU_FTR_SMT */
         * with tlbilx before overwriting.
         */
 
-       lbz     r15,TCD_ESEL_NEXT(r11)
-       rlwinm  r10,r15,16,0xff0000
+       rlwinm  r10,r7,16,0xff0000
        oris    r10,r10,MAS0_TLBSEL(1)@h
        mtspr   SPRN_MAS0,r10
        isync
@@ -429,15 +439,14 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_SMT)
        mtspr   SPRN_MAS2,r15
 
 tlb_miss_huge_done_e6500:
-       lbz     r15,TCD_ESEL_NEXT(r11)
        lbz     r16,TCD_ESEL_MAX(r11)
        lbz     r14,TCD_ESEL_FIRST(r11)
-       rlwimi  r10,r15,16,0x00ff0000   /* insert esel_next into MAS0 */
-       addi    r15,r15,1               /* increment esel_next */
+       rlwimi  r10,r7,16,0x00ff0000    /* insert esel_next into MAS0 */
+       addi    r7,r7,1                 /* increment esel_next */
        mtspr   SPRN_MAS0,r10
-       cmpw    r15,r16
-       iseleq  r15,r14,r15             /* if next == last use first */
-       stb     r15,TCD_ESEL_NEXT(r11)
+       cmpw    r7,r16
+       iseleq  r7,r14,r7               /* if next == last use first */
+       stb     r7,TCD_ESEL_NEXT(r11)
 
        tlbwe