From cb2c9b2741346eb23b177187a51ff5abf08295bd Mon Sep 17 00:00:00 2001 From: Anton Blanchard Date: Mon, 13 Feb 2006 14:48:35 +1100 Subject: [PATCH] [PATCH] powerpc: Fix runlatch performance issues The runlatch SPR can take a lot of time to write. My original runlatch code would set it on every exception entry even though most of the time this was not required. It would also continually set it in the idle loop, which is an issue on an SMT capable processor. Now we cache the runlatch value in a threadinfo bit, and only check for it in decrementer and hardware interrupt exceptions as well as the idle loop. Boot on POWER3, POWER5 and iseries, and compile tested on pmac32. Signed-off-by: Anton Blanchard Signed-off-by: Paul Mackerras --- arch/powerpc/kernel/head_64.S | 12 ++---------- arch/powerpc/kernel/process.c | 32 ++++++++++++++++++++++++++++++++ arch/powerpc/platforms/iseries/setup.c | 1 + include/asm-powerpc/reg.h | 33 +++------------------------------ include/asm-powerpc/thread_info.h | 4 ++-- 5 files changed, 40 insertions(+), 42 deletions(-) diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index 2b03a09..bb845ee 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -319,7 +319,6 @@ exception_marker: label##_pSeries: \ HMT_MEDIUM; \ mtspr SPRN_SPRG1,r13; /* save r13 */ \ - RUNLATCH_ON(r13); \ EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common) #define STD_EXCEPTION_ISERIES(n, label, area) \ @@ -327,7 +326,6 @@ label##_pSeries: \ label##_iSeries: \ HMT_MEDIUM; \ mtspr SPRN_SPRG1,r13; /* save r13 */ \ - RUNLATCH_ON(r13); \ EXCEPTION_PROLOG_ISERIES_1(area); \ EXCEPTION_PROLOG_ISERIES_2; \ b label##_common @@ -337,7 +335,6 @@ label##_iSeries: \ label##_iSeries: \ HMT_MEDIUM; \ mtspr SPRN_SPRG1,r13; /* save r13 */ \ - RUNLATCH_ON(r13); \ EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN); \ lbz r10,PACAPROCENABLED(r13); \ cmpwi 0,r10,0; \ @@ -390,6 +387,7 @@ label##_common: \ label##_common: \ EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \ DISABLE_INTS; \ + bl .ppc64_runlatch_on; \ addi r3,r1,STACK_FRAME_OVERHEAD; \ bl hdlr; \ b .ret_from_except_lite @@ -407,7 +405,6 @@ __start_interrupts: _machine_check_pSeries: HMT_MEDIUM mtspr SPRN_SPRG1,r13 /* save r13 */ - RUNLATCH_ON(r13) EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) . = 0x300 @@ -434,7 +431,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB) data_access_slb_pSeries: HMT_MEDIUM mtspr SPRN_SPRG1,r13 - RUNLATCH_ON(r13) mfspr r13,SPRN_SPRG3 /* get paca address into r13 */ std r3,PACA_EXSLB+EX_R3(r13) mfspr r3,SPRN_DAR @@ -460,7 +456,6 @@ data_access_slb_pSeries: instruction_access_slb_pSeries: HMT_MEDIUM mtspr SPRN_SPRG1,r13 - RUNLATCH_ON(r13) mfspr r13,SPRN_SPRG3 /* get paca address into r13 */ std r3,PACA_EXSLB+EX_R3(r13) mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ @@ -491,7 +486,6 @@ instruction_access_slb_pSeries: .globl system_call_pSeries system_call_pSeries: HMT_MEDIUM - RUNLATCH_ON(r9) mr r9,r13 mfmsr r10 mfspr r13,SPRN_SPRG3 @@ -575,7 +569,6 @@ slb_miss_user_pseries: system_reset_fwnmi: HMT_MEDIUM mtspr SPRN_SPRG1,r13 /* save r13 */ - RUNLATCH_ON(r13) EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common) .globl machine_check_fwnmi @@ -583,7 +576,6 @@ system_reset_fwnmi: machine_check_fwnmi: HMT_MEDIUM mtspr SPRN_SPRG1,r13 /* save r13 */ - RUNLATCH_ON(r13) EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) #ifdef CONFIG_PPC_ISERIES @@ -894,7 +886,6 @@ unrecov_fer: .align 7 .globl data_access_common data_access_common: - RUNLATCH_ON(r10) /* It wont fit in the 0x300 handler */ mfspr r10,SPRN_DAR std r10,PACA_EXGEN+EX_DAR(r13) mfspr r10,SPRN_DSISR @@ -1042,6 +1033,7 @@ hardware_interrupt_common: EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN) hardware_interrupt_entry: DISABLE_INTS + bl .ppc64_runlatch_on addi r3,r1,STACK_FRAME_OVERHEAD bl .do_IRQ b .ret_from_except_lite diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 5770399..c225cf1 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -888,3 +888,35 @@ void dump_stack(void) show_stack(current, NULL); } EXPORT_SYMBOL(dump_stack); + +#ifdef CONFIG_PPC64 +void ppc64_runlatch_on(void) +{ + unsigned long ctrl; + + if (cpu_has_feature(CPU_FTR_CTRL) && !test_thread_flag(TIF_RUNLATCH)) { + HMT_medium(); + + ctrl = mfspr(SPRN_CTRLF); + ctrl |= CTRL_RUNLATCH; + mtspr(SPRN_CTRLT, ctrl); + + set_thread_flag(TIF_RUNLATCH); + } +} + +void ppc64_runlatch_off(void) +{ + unsigned long ctrl; + + if (cpu_has_feature(CPU_FTR_CTRL) && test_thread_flag(TIF_RUNLATCH)) { + HMT_medium(); + + clear_thread_flag(TIF_RUNLATCH); + + ctrl = mfspr(SPRN_CTRLF); + ctrl &= ~CTRL_RUNLATCH; + mtspr(SPRN_CTRLT, ctrl); + } +} +#endif diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c index 3f87901..3ecc4a6 100644 --- a/arch/powerpc/platforms/iseries/setup.c +++ b/arch/powerpc/platforms/iseries/setup.c @@ -648,6 +648,7 @@ static void yield_shared_processor(void) * here and let the timer_interrupt code sort out the actual time. */ get_lppaca()->int_dword.fields.decr_int = 1; + ppc64_runlatch_on(); process_iSeries_events(); } diff --git a/include/asm-powerpc/reg.h b/include/asm-powerpc/reg.h index 12ecc9b..72bfe3a 100644 --- a/include/asm-powerpc/reg.h +++ b/include/asm-powerpc/reg.h @@ -615,27 +615,9 @@ #define proc_trap() asm volatile("trap") #ifdef CONFIG_PPC64 -static inline void ppc64_runlatch_on(void) -{ - unsigned long ctrl; - - if (cpu_has_feature(CPU_FTR_CTRL)) { - ctrl = mfspr(SPRN_CTRLF); - ctrl |= CTRL_RUNLATCH; - mtspr(SPRN_CTRLT, ctrl); - } -} - -static inline void ppc64_runlatch_off(void) -{ - unsigned long ctrl; - - if (cpu_has_feature(CPU_FTR_CTRL)) { - ctrl = mfspr(SPRN_CTRLF); - ctrl &= ~CTRL_RUNLATCH; - mtspr(SPRN_CTRLT, ctrl); - } -} + +extern void ppc64_runlatch_on(void); +extern void ppc64_runlatch_off(void); extern unsigned long scom970_read(unsigned int address); extern void scom970_write(unsigned int address, unsigned long value); @@ -645,15 +627,6 @@ extern void scom970_write(unsigned int address, unsigned long value); #define __get_SP() ({unsigned long sp; \ asm volatile("mr %0,1": "=r" (sp)); sp;}) -#else /* __ASSEMBLY__ */ - -#define RUNLATCH_ON(REG) \ -BEGIN_FTR_SECTION \ - mfspr (REG),SPRN_CTRLF; \ - ori (REG),(REG),CTRL_RUNLATCH; \ - mtspr SPRN_CTRLT,(REG); \ -END_FTR_SECTION_IFSET(CPU_FTR_CTRL) - #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_REG_H */ diff --git a/include/asm-powerpc/thread_info.h b/include/asm-powerpc/thread_info.h index c044ec1..237fc2b 100644 --- a/include/asm-powerpc/thread_info.h +++ b/include/asm-powerpc/thread_info.h @@ -113,7 +113,7 @@ static inline struct thread_info *current_thread_info(void) #define TIF_POLLING_NRFLAG 4 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_32BIT 5 /* 32 bit binary */ -/* #define SPARE 6 */ +#define TIF_RUNLATCH 6 /* Is the runlatch enabled? */ #define TIF_ABI_PENDING 7 /* 32/64 bit switch needed */ #define TIF_SYSCALL_AUDIT 8 /* syscall auditing active */ #define TIF_SINGLESTEP 9 /* singlestepping active */ @@ -131,7 +131,7 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_NEED_RESCHED (1<