powerpc/tm: Switch out userspace PPR and DSCR sooner

When we do a treclaim or trecheckpoint we end up running with userspace
PPR and DSCR values.  Currently we don't do anything special to avoid
running with user values which could cause a severe performance
degradation.

This patch moves the PPR and DSCR save and restore around treclaim and
trecheckpoint so that we run with user values for a much shorter period.
More care is taken with the PPR as it's impact is greater than the DSCR.

This is similar to user exceptions, where we run HTM_MEDIUM early to
ensure that we don't run with a userspace PPR values in the kernel.

Signed-off-by: Michael Neuling <mikey@neuling.org>
Cc: <stable@vger.kernel.org> # 3.9+
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

diff --git a/arch/powerpc/kernel/tm.S b/arch/powerpc/kernel/tm.S
index 8ece190..cd809ea 100644 (file)
--- a/arch/powerpc/kernel/tm.S
+++ b/arch/powerpc/kernel/tm.S
@@ -79,6 +79,11 @@ _GLOBAL(tm_abort)
        TABORT(R3)
        blr
 
+       .section        ".toc","aw"
+DSCR_DEFAULT:
+       .tc dscr_default[TC],dscr_default
+
+       .section        ".text"
 
 /* void tm_reclaim(struct thread_struct *thread,
  *                 unsigned long orig_msr,
@@ -188,11 +193,18 @@ dont_backup_fp:
        std     r1, PACATMSCRATCH(r13)
        ld      r1, PACAR1(r13)
 
+       /* Store the PPR in r11 and reset to decent value */
+       std     r11, GPR11(r1)                  /* Temporary stash */
+       mfspr   r11, SPRN_PPR
+       HMT_MEDIUM
+
        /* Now get some more GPRS free */
        std     r7, GPR7(r1)                    /* Temporary stash */
        std     r12, GPR12(r1)                  /* ''   ''    ''   */
        ld      r12, STACK_PARAM(0)(r1)         /* Param 0, thread_struct * */
 
+       std     r11, THREAD_TM_PPR(r12)         /* Store PPR and free r11 */
+
        addi    r7, r12, PT_CKPT_REGS           /* Thread's ckpt_regs */
 
        /* Make r7 look like an exception frame so that we
@@ -204,15 +216,19 @@ dont_backup_fp:
        SAVE_GPR(0, r7)                         /* user r0 */
        SAVE_GPR(2, r7)                 /* user r2 */
        SAVE_4GPRS(3, r7)                       /* user r3-r6 */
-       SAVE_4GPRS(8, r7)                       /* user r8-r11 */
+       SAVE_GPR(8, r7)                         /* user r8 */
+       SAVE_GPR(9, r7)                         /* user r9 */
+       SAVE_GPR(10, r7)                        /* user r10 */
        ld      r3, PACATMSCRATCH(r13)          /* user r1 */
        ld      r4, GPR7(r1)                    /* user r7 */
-       ld      r5, GPR12(r1)                   /* user r12 */
-       GET_SCRATCH0(6)                         /* user r13 */
+       ld      r5, GPR11(r1)                   /* user r11 */
+       ld      r6, GPR12(r1)                   /* user r12 */
+       GET_SCRATCH0(8)                         /* user r13 */
        std     r3, GPR1(r7)
        std     r4, GPR7(r7)
-       std     r5, GPR12(r7)
-       std     r6, GPR13(r7)
+       std     r5, GPR11(r7)
+       std     r6, GPR12(r7)
+       std     r8, GPR13(r7)
 
        SAVE_NVGPRS(r7)                         /* user r14-r31 */
 
@@ -235,14 +251,12 @@ dont_backup_fp:
        std     r6, _XER(r7)
 
 
-       /* ******************** TAR, PPR, DSCR ********** */
+       /* ******************** TAR, DSCR ********** */
        mfspr   r3, SPRN_TAR
-       mfspr   r4, SPRN_PPR
-       mfspr   r5, SPRN_DSCR
+       mfspr   r4, SPRN_DSCR
 
        std     r3, THREAD_TM_TAR(r12)
-       std     r4, THREAD_TM_PPR(r12)
-       std     r5, THREAD_TM_DSCR(r12)
+       std     r4, THREAD_TM_DSCR(r12)
 
        /* MSR and flags:  We don't change CRs, and we don't need to alter
         * MSR.
@@ -259,7 +273,7 @@ dont_backup_fp:
        std     r3, THREAD_TM_TFHAR(r12)
        std     r4, THREAD_TM_TFIAR(r12)
 
-       /* AMR and PPR are checkpointed too, but are unsupported by Linux. */
+       /* AMR is checkpointed too, but is unsupported by Linux. */
 
        /* Restore original MSR/IRQ state & clear TM mode */
        ld      r14, TM_FRAME_L0(r1)            /* Orig MSR */
@@ -275,6 +289,12 @@ dont_backup_fp:
        mtcr    r4
        mtlr    r0
        ld      r2, 40(r1)
+
+       /* Load system default DSCR */
+       ld      r4, DSCR_DEFAULT@toc(r2)
+       ld      r0, 0(r4)
+       mtspr   SPRN_DSCR, r0
+
        blr
 
 
@@ -359,25 +379,24 @@ dont_restore_fp:
 
 restore_gprs:
 
-       /* ******************** TAR, PPR, DSCR ********** */
-       ld      r4, THREAD_TM_TAR(r3)
-       ld      r5, THREAD_TM_PPR(r3)
-       ld      r6, THREAD_TM_DSCR(r3)
+       /* ******************** CR,LR,CCR,MSR ********** */
+       ld      r4, _CTR(r7)
+       ld      r5, _LINK(r7)
+       ld      r6, _CCR(r7)
+       ld      r8, _XER(r7)
 
-       mtspr   SPRN_TAR,       r4
-       mtspr   SPRN_PPR,       r5
-       mtspr   SPRN_DSCR,      r6
+       mtctr   r4
+       mtlr    r5
+       mtcr    r6
+       mtxer   r8
 
-       /* ******************** CR,LR,CCR,MSR ********** */
-       ld      r3, _CTR(r7)
-       ld      r4, _LINK(r7)
-       ld      r5, _CCR(r7)
-       ld      r6, _XER(r7)
+       /* ******************** TAR ******************** */
+       ld      r4, THREAD_TM_TAR(r3)
+       mtspr   SPRN_TAR,       r4
 
-       mtctr   r3
-       mtlr    r4
-       mtcr    r5
-       mtxer   r6
+       /* Load up the PPR and DSCR in GPRs only at this stage */
+       ld      r5, THREAD_TM_DSCR(r3)
+       ld      r6, THREAD_TM_PPR(r3)
 
        /* Clear the MSR RI since we are about to change R1.  EE is already off
         */
@@ -385,19 +404,26 @@ restore_gprs:
        mtmsrd  r4, 1
 
        REST_4GPRS(0, r7)                       /* GPR0-3 */
-       REST_GPR(4, r7)                         /* GPR4-6 */
-       REST_GPR(5, r7)
-       REST_GPR(6, r7)
+       REST_GPR(4, r7)                         /* GPR4 */
        REST_4GPRS(8, r7)                       /* GPR8-11 */
        REST_2GPRS(12, r7)                      /* GPR12-13 */
 
        REST_NVGPRS(r7)                         /* GPR14-31 */
 
-       ld      r7, GPR7(r7)                    /* GPR7 */
+       /* Load up PPR and DSCR here so we don't run with user values for long
+        */
+       mtspr   SPRN_DSCR, r5
+       mtspr   SPRN_PPR, r6
+
+       REST_GPR(5, r7)                         /* GPR5-7 */
+       REST_GPR(6, r7)
+       ld      r7, GPR7(r7)
 
        /* Commit register state as checkpointed state: */
        TRECHKPT
 
+       HMT_MEDIUM
+
        /* Our transactional state has now changed.
         *
         * Now just get out of here.  Transactional (current) state will be
@@ -420,6 +446,12 @@ restore_gprs:
        mtcr    r4
        mtlr    r0
        ld      r2, 40(r1)
+
+       /* Load system default DSCR */
+       ld      r4, DSCR_DEFAULT@toc(r2)
+       ld      r0, 0(r4)
+       mtspr   SPRN_DSCR, r0
+
        blr
 
        /* ****************************************************************** */