static void tm_reclaim_thread(struct thread_struct *thr,
struct thread_info *ti, uint8_t cause)
{
- unsigned long msr_diff = 0;
-
- /*
- * If FP/VSX registers have been already saved to the
- * thread_struct, move them to the transact_fp array.
- * We clear the TIF_RESTORE_TM bit since after the reclaim
- * the thread will no longer be transactional.
- */
- if (test_ti_thread_flag(ti, TIF_RESTORE_TM)) {
- msr_diff = thr->ckpt_regs.msr & ~thr->regs->msr;
- if (msr_diff & MSR_FP)
- memcpy(&thr->transact_fp, &thr->fp_state,
- sizeof(struct thread_fp_state));
- if (msr_diff & MSR_VEC)
- memcpy(&thr->transact_vr, &thr->vr_state,
- sizeof(struct thread_vr_state));
- clear_ti_thread_flag(ti, TIF_RESTORE_TM);
- msr_diff &= MSR_FP | MSR_VEC | MSR_VSX | MSR_FE0 | MSR_FE1;
- }
-
/*
* Use the current MSR TM suspended bit to track if we have
* checkpointed state outstanding.
if (!MSR_TM_SUSPENDED(mfmsr()))
return;
- tm_reclaim(thr, thr->regs->msr, cause);
+ giveup_all(container_of(thr, struct task_struct, thread));
- /* Having done the reclaim, we now have the checkpointed
- * FP/VSX values in the registers. These might be valid
- * even if we have previously called enable_kernel_fp() or
- * flush_fp_to_thread(), so update thr->regs->msr to
- * indicate their current validity.
- */
- thr->regs->msr |= msr_diff;
+ tm_reclaim(thr, thr->ckpt_regs.msr, cause);
}
void tm_reclaim_current(uint8_t cause)
if (!MSR_TM_ACTIVE(thr->regs->msr))
goto out_and_saveregs;
- /* Stash the original thread MSR, as giveup_fpu et al will
- * modify it. We hold onto it to see whether the task used
- * FP & vector regs. If the TIF_RESTORE_TM flag is set,
- * ckpt_regs.msr is already set.
- */
- if (!test_ti_thread_flag(task_thread_info(tsk), TIF_RESTORE_TM))
- thr->ckpt_regs.msr = thr->regs->msr;
-
TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, "
"ccr=%lx, msr=%lx, trap=%lx)\n",
tsk->pid, thr->regs->nip,
* If the task was using FP, we non-lazily reload both the original and
* the speculative FP register states. This is because the kernel
* doesn't see if/when a TM rollback occurs, so if we take an FP
- * unavoidable later, we are unable to determine which set of FP regs
+ * unavailable later, we are unable to determine which set of FP regs
* need to be restored.
*/
if (!new->thread.regs)
"(new->msr 0x%lx, new->origmsr 0x%lx)\n",
new->pid, new->thread.regs->msr, msr);
- /* This loads the checkpointed FP/VEC state, if used */
tm_recheckpoint(&new->thread, msr);
- /* This loads the speculative FP/VEC state, if used */
- if (msr & MSR_FP) {
- do_load_up_transact_fpu(&new->thread);
- new->thread.regs->msr |=
- (MSR_FP | new->thread.fpexc_mode);
- }
-#ifdef CONFIG_ALTIVEC
- if (msr & MSR_VEC) {
- do_load_up_transact_altivec(&new->thread);
- new->thread.regs->msr |= MSR_VEC;
- }
-#endif
- /* We may as well turn on VSX too since all the state is restored now */
- if (msr & MSR_VSX)
- new->thread.regs->msr |= MSR_VSX;
+ /*
+ * The checkpointed state has been restored but the live state has
+ * not, ensure all the math functionality is turned off to trigger
+ * restore_math() to reload.
+ */
+ new->thread.regs->msr &= ~(MSR_FP | MSR_VEC | MSR_VSX);
TM_DEBUG("*** tm_recheckpoint of pid %d complete "
"(kernel msr 0x%lx)\n",
new->pid, mfmsr());
}
-static inline void __switch_to_tm(struct task_struct *prev)
+static inline void __switch_to_tm(struct task_struct *prev,
+ struct task_struct *new)
{
if (cpu_has_feature(CPU_FTR_TM)) {
tm_enable();
tm_reclaim_task(prev);
+ tm_recheckpoint_new_task(new);
}
}
{
unsigned long msr_diff;
+ /*
+ * This is the only moment we should clear TIF_RESTORE_TM as
+ * it is here that ckpt_regs.msr and pt_regs.msr become the same
+ * again, anything else could lead to an incorrect ckpt_msr being
+ * saved and therefore incorrect signal contexts.
+ */
clear_thread_flag(TIF_RESTORE_TM);
if (!MSR_TM_ACTIVE(regs->msr))
return;
#else
#define tm_recheckpoint_new_task(new)
-#define __switch_to_tm(prev)
+#define __switch_to_tm(prev, new)
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
static inline void save_sprs(struct thread_struct *t)
*/
save_sprs(&prev->thread);
- __switch_to_tm(prev);
-
/* Save FPU, Altivec, VSX and SPE state */
giveup_all(prev);
+ __switch_to_tm(prev, new);
+
/*
* We can't take a PMU exception inside _switch() since there is a
* window where the kernel stack SLB and the kernel stack are out
*/
hard_irq_disable();
- tm_recheckpoint_new_task(new);
-
/*
* Call restore_sprs() before calling _switch(). If we move it after
* _switch() then we miss out on calling it for new tasks. The reason
* tm_recheckpoint_new_task() (on the same task) to restore the
* checkpointed state back and the TM mode.
*/
- __switch_to_tm(src);
- tm_recheckpoint_new_task(src);
+ __switch_to_tm(src, src);
*dst = *src;
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which returns the current running values of
- * all the FPR registers, needs to know whether any transaction is active
- * or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'transact_fp' holds the last checkpointed
+ * value of all FPR registers for the current transaction.
*
* Userspace interface buffer layout:
*
* u64 fpr[32];
* u64 fpscr;
* };
- *
- * There are two config options CONFIG_VSX and CONFIG_PPC_TRANSACTIONAL_MEM
- * which determines the final code in this function. All the combinations of
- * these two config options are possible except the one below as transactional
- * memory config pulls in CONFIG_VSX automatically.
- *
- * !defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
*/
static int fpr_get(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
#ifdef CONFIG_VSX
u64 buf[33];
int i;
-#endif
- flush_fp_to_thread(target);
-#if defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
- /* copy to local buffer then write that out */
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.TS_TRANS_FPR(i);
- buf[32] = target->thread.transact_fp.fpscr;
- } else {
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.TS_FPR(i);
- buf[32] = target->thread.fp_state.fpscr;
- }
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
-#endif
+ flush_fp_to_thread(target);
-#if defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
/* copy to local buffer then write that out */
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.TS_FPR(i);
buf[32] = target->thread.fp_state.fpscr;
return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
-#endif
-
-#if !defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
+#else
BUILD_BUG_ON(offsetof(struct thread_fp_state, fpscr) !=
offsetof(struct thread_fp_state, fpr[32]));
+ flush_fp_to_thread(target);
+
return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.fp_state, 0, -1);
#endif
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which setss the current running values of
- * all the FPR registers, needs to know whether any transaction is active
- * or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'transact_fp' holds the last checkpointed
+ * value of all FPR registers for the current transaction.
*
* Userspace interface buffer layout:
*
* u64 fpscr;
* };
*
- * There are two config options CONFIG_VSX and CONFIG_PPC_TRANSACTIONAL_MEM
- * which determines the final code in this function. All the combinations of
- * these two config options are possible except the one below as transactional
- * memory config pulls in CONFIG_VSX automatically.
- *
- * !defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
*/
static int fpr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
#ifdef CONFIG_VSX
u64 buf[33];
int i;
-#endif
+
flush_fp_to_thread(target);
-#if defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
/* copy to local buffer then write that out */
i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
if (i)
return i;
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
- for (i = 0; i < 32 ; i++)
- target->thread.TS_TRANS_FPR(i) = buf[i];
- target->thread.transact_fp.fpscr = buf[32];
- } else {
- for (i = 0; i < 32 ; i++)
- target->thread.TS_FPR(i) = buf[i];
- target->thread.fp_state.fpscr = buf[32];
- }
- return 0;
-#endif
-
-#if defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
- /* copy to local buffer then write that out */
- i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
- if (i)
- return i;
for (i = 0; i < 32 ; i++)
target->thread.TS_FPR(i) = buf[i];
target->thread.fp_state.fpscr = buf[32];
return 0;
-#endif
-
-#if !defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
+#else
BUILD_BUG_ON(offsetof(struct thread_fp_state, fpscr) !=
offsetof(struct thread_fp_state, fpr[32]));
+ flush_fp_to_thread(target);
+
return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.fp_state, 0, -1);
#endif
}
/*
- * When the transaction is active, 'transact_vr' holds the current running
- * value of all the VMX registers and 'vr_state' holds the last checkpointed
- * value of all the VMX registers for the current transaction to fall back
- * on in case it aborts. When transaction is not active 'vr_state' holds
- * the current running state of all the VMX registers. So this function which
- * gets the current running values of all the VMX registers, needs to know
- * whether any transaction is active or not.
+ * Regardless of transactions, 'vr_state' holds the current running
+ * value of all the VMX registers and 'transact_vr' holds the last
+ * checkpointed value of all the VMX registers for the current
+ * transaction to fall back on in case it aborts.
*
* Userspace interface buffer layout:
*
unsigned int pos, unsigned int count,
void *kbuf, void __user *ubuf)
{
- struct thread_vr_state *addr;
int ret;
flush_altivec_to_thread(target);
BUILD_BUG_ON(offsetof(struct thread_vr_state, vscr) !=
offsetof(struct thread_vr_state, vr[32]));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_fp_to_thread(target);
- flush_tmregs_to_thread(target);
- addr = &target->thread.transact_vr;
- } else {
- addr = &target->thread.vr_state;
- }
-#else
- addr = &target->thread.vr_state;
-#endif
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- addr, 0,
+ &target->thread.vr_state, 0,
33 * sizeof(vector128));
if (!ret) {
/*
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr))
- vrsave.word = target->thread.transact_vrsave;
- else
- vrsave.word = target->thread.vrsave;
-#else
vrsave.word = target->thread.vrsave;
-#endif
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
}
/*
- * When the transaction is active, 'transact_vr' holds the current running
- * value of all the VMX registers and 'vr_state' holds the last checkpointed
- * value of all the VMX registers for the current transaction to fall back
- * on in case it aborts. When transaction is not active 'vr_state' holds
- * the current running state of all the VMX registers. So this function which
- * sets the current running values of all the VMX registers, needs to know
- * whether any transaction is active or not.
+ * Regardless of transactions, 'vr_state' holds the current running
+ * value of all the VMX registers and 'transact_vr' holds the last
+ * checkpointed value of all the VMX registers for the current
+ * transaction to fall back on in case it aborts.
*
* Userspace interface buffer layout:
*
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf)
{
- struct thread_vr_state *addr;
int ret;
flush_altivec_to_thread(target);
BUILD_BUG_ON(offsetof(struct thread_vr_state, vscr) !=
offsetof(struct thread_vr_state, vr[32]));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_fp_to_thread(target);
- flush_tmregs_to_thread(target);
- addr = &target->thread.transact_vr;
- } else {
- addr = &target->thread.vr_state;
- }
-#else
- addr = &target->thread.vr_state;
-#endif
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- addr, 0,
+ &target->thread.vr_state, 0,
33 * sizeof(vector128));
if (!ret && count > 0) {
/*
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr))
- vrsave.word = target->thread.transact_vrsave;
- else
- vrsave.word = target->thread.vrsave;
-#else
vrsave.word = target->thread.vrsave;
-#endif
+
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
- if (!ret) {
-
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr))
- target->thread.transact_vrsave = vrsave.word;
- else
- target->thread.vrsave = vrsave.word;
-#else
+ if (!ret)
target->thread.vrsave = vrsave.word;
-#endif
- }
}
return ret;
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which returns the current running values of
- * all the FPR registers, needs to know whether any transaction is active
- * or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'transact_fp' holds the last
+ * checkpointed value of all FPR registers for the current
+ * transaction.
*
* Userspace interface buffer layout:
*
u64 buf[32];
int ret, i;
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
-#endif
flush_vsx_to_thread(target);
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.
- transact_fp.fpr[i][TS_VSRLOWOFFSET];
- } else {
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.
- fp_state.fpr[i][TS_VSRLOWOFFSET];
- }
-#else
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET];
-#endif
+
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which sets the current running values of all
- * the FPR registers, needs to know whether any transaction is active or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'transact_fp' holds the last
+ * checkpointed value of all FPR registers for the current
+ * transaction.
*
* Userspace interface buffer layout:
*
u64 buf[32];
int ret,i;
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
-#endif
flush_vsx_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
-
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- for (i = 0; i < 32 ; i++)
- target->thread.transact_fp.
- fpr[i][TS_VSRLOWOFFSET] = buf[i];
- } else {
+ if (!ret)
for (i = 0; i < 32 ; i++)
- target->thread.fp_state.
- fpr[i][TS_VSRLOWOFFSET] = buf[i];
- }
-#else
- for (i = 0; i < 32 ; i++)
- target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
-#endif
-
+ target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
return ret;
}
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.ckpt_regs,
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.ckpt_regs,
*
* This function gets in transaction checkpointed FPR registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'transact_fp' holds the checkpointed
* values for the current transaction to fall back on if it aborts
* in between. This function gets those checkpointed FPR registers.
* The userspace interface buffer layout is as follows.
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* copy to local buffer then write that out */
for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.TS_FPR(i);
- buf[32] = target->thread.fp_state.fpscr;
+ buf[i] = target->thread.TS_TRANS_FPR(i);
+ buf[32] = target->thread.transact_fp.fpscr;
return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
}
*
* This function sets in transaction checkpointed FPR registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'transact_fp' holds the checkpointed
* FPR register values for the current transaction to fall back on
* if it aborts in between. This function sets these checkpointed
* FPR registers. The userspace interface buffer layout is as follows.
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* copy to local buffer then write that out */
i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
if (i)
return i;
for (i = 0; i < 32 ; i++)
- target->thread.TS_FPR(i) = buf[i];
- target->thread.fp_state.fpscr = buf[32];
+ target->thread.TS_TRANS_FPR(i) = buf[i];
+ target->thread.transact_fp.fpscr = buf[32];
return 0;
}
*
* This function gets in transaction checkpointed VMX registers.
*
- * When the transaction is active 'vr_state' and 'vr_save' hold
+ * When the transaction is active 'transact_vr' and 'transact_vrsave' hold
* the checkpointed values for the current transaction to fall
* back on if it aborts in between. The userspace interface buffer
* layout is as follows.
return -ENODATA;
/* Flush the state */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.vr_state, 0,
+ &target->thread.transact_vr, 0,
33 * sizeof(vector128));
if (!ret) {
/*
u32 word;
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
- vrsave.word = target->thread.vrsave;
+ vrsave.word = target->thread.transact_vrsave;
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
}
*
* This function sets in transaction checkpointed VMX registers.
*
- * When the transaction is active 'vr_state' and 'vr_save' hold
+ * When the transaction is active 'transact_vr' and 'transact_vrsave' hold
* the checkpointed values for the current transaction to fall
* back on if it aborts in between. The userspace interface buffer
* layout is as follows.
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- &target->thread.vr_state, 0,
+ &target->thread.transact_vr, 0,
33 * sizeof(vector128));
if (!ret && count > 0) {
/*
u32 word;
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
- vrsave.word = target->thread.vrsave;
+ vrsave.word = target->thread.transact_vrsave;
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
if (!ret)
- target->thread.vrsave = vrsave.word;
+ target->thread.transact_vrsave = vrsave.word;
}
return ret;
*
* This function gets in transaction checkpointed VSX registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'transact_fp' holds the checkpointed
* values for the current transaction to fall back on if it aborts
* in between. This function gets those checkpointed VSX registers.
* The userspace interface buffer layout is as follows.
return -ENODATA;
/* Flush the state */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
flush_vsx_to_thread(target);
for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET];
+ buf[i] = target->thread.transact_fp.fpr[i][TS_VSRLOWOFFSET];
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
*
* This function sets in transaction checkpointed VSX registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'transact_fp' holds the checkpointed
* VSX register values for the current transaction to fall back on
* if it aborts in between. This function sets these checkpointed
* FPR registers. The userspace interface buffer layout is as follows.
return -ENODATA;
/* Flush the state */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
flush_vsx_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
- for (i = 0; i < 32 ; i++)
- target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
+ if (!ret)
+ for (i = 0; i < 32 ; i++)
+ target->thread.transact_fp.fpr[i][TS_VSRLOWOFFSET] = buf[i];
return ret;
}
return -ENODEV;
/* Flush the states */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* TFHAR register */
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
return -ENODEV;
/* Flush the states */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* TFHAR register */
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
*/
regs->msr &= ~MSR_TS_MASK;
- /* Make sure floating point registers are stored in regs */
- flush_fp_to_thread(current);
-
/* Save both sets of general registers */
if (save_general_regs(¤t->thread.ckpt_regs, frame)
|| save_general_regs(regs, tm_frame))
#ifdef CONFIG_ALTIVEC
/* save altivec registers */
if (current->thread.used_vr) {
- flush_altivec_to_thread(current);
- if (__copy_to_user(&frame->mc_vregs, ¤t->thread.vr_state,
+ if (__copy_to_user(&frame->mc_vregs, ¤t->thread.transact_vr,
ELF_NVRREG * sizeof(vector128)))
return 1;
if (msr & MSR_VEC) {
if (__copy_to_user(&tm_frame->mc_vregs,
- ¤t->thread.transact_vr,
+ ¤t->thread.vr_state,
ELF_NVRREG * sizeof(vector128)))
return 1;
} else {
if (__copy_to_user(&tm_frame->mc_vregs,
- ¤t->thread.vr_state,
+ ¤t->thread.transact_vr,
ELF_NVRREG * sizeof(vector128)))
return 1;
}
* most significant bits of that same vector. --BenH
*/
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- current->thread.vrsave = mfspr(SPRN_VRSAVE);
- if (__put_user(current->thread.vrsave,
+ current->thread.transact_vrsave = mfspr(SPRN_VRSAVE);
+ if (__put_user(current->thread.transact_vrsave,
(u32 __user *)&frame->mc_vregs[32]))
return 1;
if (msr & MSR_VEC) {
- if (__put_user(current->thread.transact_vrsave,
+ if (__put_user(current->thread.vrsave,
(u32 __user *)&tm_frame->mc_vregs[32]))
return 1;
} else {
- if (__put_user(current->thread.vrsave,
+ if (__put_user(current->thread.transact_vrsave,
(u32 __user *)&tm_frame->mc_vregs[32]))
return 1;
}
#endif /* CONFIG_ALTIVEC */
- if (copy_fpr_to_user(&frame->mc_fregs, current))
+ if (copy_transact_fpr_to_user(&frame->mc_fregs, current))
return 1;
if (msr & MSR_FP) {
- if (copy_transact_fpr_to_user(&tm_frame->mc_fregs, current))
+ if (copy_fpr_to_user(&tm_frame->mc_fregs, current))
return 1;
} else {
- if (copy_fpr_to_user(&tm_frame->mc_fregs, current))
+ if (copy_transact_fpr_to_user(&tm_frame->mc_fregs, current))
return 1;
}
* contains valid data
*/
if (current->thread.used_vsr) {
- flush_vsx_to_thread(current);
- if (copy_vsx_to_user(&frame->mc_vsregs, current))
+ if (copy_transact_vsx_to_user(&frame->mc_vsregs, current))
return 1;
if (msr & MSR_VSX) {
- if (copy_transact_vsx_to_user(&tm_frame->mc_vsregs,
+ if (copy_vsx_to_user(&tm_frame->mc_vsregs,
current))
return 1;
} else {
- if (copy_vsx_to_user(&tm_frame->mc_vsregs, current))
+ if (copy_transact_vsx_to_user(&tm_frame->mc_vsregs, current))
return 1;
}
regs->msr &= ~MSR_VEC;
if (msr & MSR_VEC) {
/* restore altivec registers from the stack */
- if (__copy_from_user(¤t->thread.vr_state, &sr->mc_vregs,
+ if (__copy_from_user(¤t->thread.transact_vr, &sr->mc_vregs,
sizeof(sr->mc_vregs)) ||
- __copy_from_user(¤t->thread.transact_vr,
+ __copy_from_user(¤t->thread.vr_state,
&tm_sr->mc_vregs,
sizeof(sr->mc_vregs)))
return 1;
}
/* Always get VRSAVE back */
- if (__get_user(current->thread.vrsave,
+ if (__get_user(current->thread.transact_vrsave,
(u32 __user *)&sr->mc_vregs[32]) ||
- __get_user(current->thread.transact_vrsave,
+ __get_user(current->thread.vrsave,
(u32 __user *)&tm_sr->mc_vregs[32]))
return 1;
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- mtspr(SPRN_VRSAVE, current->thread.vrsave);
+ mtspr(SPRN_VRSAVE, current->thread.transact_vrsave);
#endif /* CONFIG_ALTIVEC */
regs->msr &= ~(MSR_FP | MSR_FE0 | MSR_FE1);
* Restore altivec registers from the stack to a local
* buffer, then write this out to the thread_struct
*/
- if (copy_vsx_from_user(current, &sr->mc_vsregs) ||
- copy_transact_vsx_from_user(current, &tm_sr->mc_vsregs))
+ if (copy_vsx_from_user(current, &tm_sr->mc_vsregs) ||
+ copy_transact_vsx_from_user(current, &sr->mc_vsregs))
return 1;
current->thread.used_vsr = true;
} else if (current->thread.used_vsr)
tm_recheckpoint(¤t->thread, msr);
/* This loads the speculative FP/VEC state, if used */
+ msr_check_and_set(msr & (MSR_FP | MSR_VEC));
if (msr & MSR_FP) {
- do_load_up_transact_fpu(¤t->thread);
+ load_fp_state(¤t->thread.fp_state);
regs->msr |= (MSR_FP | current->thread.fpexc_mode);
}
#ifdef CONFIG_ALTIVEC
if (msr & MSR_VEC) {
- do_load_up_transact_altivec(¤t->thread);
+ load_vr_state(¤t->thread.vr_state);
regs->msr |= MSR_VEC;
}
#endif
*/
regs->msr &= ~MSR_TS_MASK;
- flush_fp_to_thread(tsk);
-
#ifdef CONFIG_ALTIVEC
err |= __put_user(v_regs, &sc->v_regs);
err |= __put_user(tm_v_regs, &tm_sc->v_regs);
/* save altivec registers */
if (tsk->thread.used_vr) {
- flush_altivec_to_thread(tsk);
/* Copy 33 vec registers (vr0..31 and vscr) to the stack */
- err |= __copy_to_user(v_regs, &tsk->thread.vr_state,
+ err |= __copy_to_user(v_regs, &tsk->thread.transact_vr,
33 * sizeof(vector128));
/* If VEC was enabled there are transactional VRs valid too,
* else they're a copy of the checkpointed VRs.
*/
if (msr & MSR_VEC)
err |= __copy_to_user(tm_v_regs,
- &tsk->thread.transact_vr,
+ &tsk->thread.vr_state,
33 * sizeof(vector128));
else
err |= __copy_to_user(tm_v_regs,
- &tsk->thread.vr_state,
+ &tsk->thread.transact_vr,
33 * sizeof(vector128));
/* set MSR_VEC in the MSR value in the frame to indicate
* use altivec.
*/
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- tsk->thread.vrsave = mfspr(SPRN_VRSAVE);
- err |= __put_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33]);
+ tsk->thread.transact_vrsave = mfspr(SPRN_VRSAVE);
+ err |= __put_user(tsk->thread.transact_vrsave, (u32 __user *)&v_regs[33]);
if (msr & MSR_VEC)
- err |= __put_user(tsk->thread.transact_vrsave,
+ err |= __put_user(tsk->thread.vrsave,
(u32 __user *)&tm_v_regs[33]);
else
- err |= __put_user(tsk->thread.vrsave,
+ err |= __put_user(tsk->thread.transact_vrsave,
(u32 __user *)&tm_v_regs[33]);
#else /* CONFIG_ALTIVEC */
#endif /* CONFIG_ALTIVEC */
/* copy fpr regs and fpscr */
- err |= copy_fpr_to_user(&sc->fp_regs, tsk);
+ err |= copy_transact_fpr_to_user(&sc->fp_regs, tsk);
if (msr & MSR_FP)
- err |= copy_transact_fpr_to_user(&tm_sc->fp_regs, tsk);
- else
err |= copy_fpr_to_user(&tm_sc->fp_regs, tsk);
+ else
+ err |= copy_transact_fpr_to_user(&tm_sc->fp_regs, tsk);
#ifdef CONFIG_VSX
/*
* VMX data.
*/
if (tsk->thread.used_vsr) {
- flush_vsx_to_thread(tsk);
v_regs += ELF_NVRREG;
tm_v_regs += ELF_NVRREG;
- err |= copy_vsx_to_user(v_regs, tsk);
+ err |= copy_transact_vsx_to_user(v_regs, tsk);
if (msr & MSR_VSX)
- err |= copy_transact_vsx_to_user(tm_v_regs, tsk);
- else
err |= copy_vsx_to_user(tm_v_regs, tsk);
+ else
+ err |= copy_transact_vsx_to_user(tm_v_regs, tsk);
/* set MSR_VSX in the MSR value in the frame to
* indicate that sc->vs_reg) contains valid data.
return -EFAULT;
/* Copy 33 vec registers (vr0..31 and vscr) from the stack */
if (v_regs != NULL && tm_v_regs != NULL && (msr & MSR_VEC) != 0) {
- err |= __copy_from_user(&tsk->thread.vr_state, v_regs,
+ err |= __copy_from_user(&tsk->thread.transact_vr, v_regs,
33 * sizeof(vector128));
- err |= __copy_from_user(&tsk->thread.transact_vr, tm_v_regs,
+ err |= __copy_from_user(&tsk->thread.vr_state, tm_v_regs,
33 * sizeof(vector128));
current->thread.used_vr = true;
}
}
/* Always get VRSAVE back */
if (v_regs != NULL && tm_v_regs != NULL) {
- err |= __get_user(tsk->thread.vrsave,
- (u32 __user *)&v_regs[33]);
err |= __get_user(tsk->thread.transact_vrsave,
+ (u32 __user *)&v_regs[33]);
+ err |= __get_user(tsk->thread.vrsave,
(u32 __user *)&tm_v_regs[33]);
}
else {
mtspr(SPRN_VRSAVE, tsk->thread.vrsave);
#endif /* CONFIG_ALTIVEC */
/* restore floating point */
- err |= copy_fpr_from_user(tsk, &sc->fp_regs);
- err |= copy_transact_fpr_from_user(tsk, &tm_sc->fp_regs);
+ err |= copy_fpr_from_user(tsk, &tm_sc->fp_regs);
+ err |= copy_transact_fpr_from_user(tsk, &sc->fp_regs);
#ifdef CONFIG_VSX
/*
* Get additional VSX data. Update v_regs to point after the
if (v_regs && ((msr & MSR_VSX) != 0)) {
v_regs += ELF_NVRREG;
tm_v_regs += ELF_NVRREG;
- err |= copy_vsx_from_user(tsk, v_regs);
- err |= copy_transact_vsx_from_user(tsk, tm_v_regs);
+ err |= copy_vsx_from_user(tsk, tm_v_regs);
+ err |= copy_transact_vsx_from_user(tsk, v_regs);
tsk->thread.used_vsr = true;
} else {
for (i = 0; i < 32 ; i++) {
/* This loads the checkpointed FP/VEC state, if used */
tm_recheckpoint(&tsk->thread, msr);
- /* This loads the speculative FP/VEC state, if used */
+ msr_check_and_set(msr & (MSR_FP | MSR_VEC));
if (msr & MSR_FP) {
- do_load_up_transact_fpu(&tsk->thread);
+ load_fp_state(&tsk->thread.fp_state);
regs->msr |= (MSR_FP | tsk->thread.fpexc_mode);
}
-#ifdef CONFIG_ALTIVEC
if (msr & MSR_VEC) {
- do_load_up_transact_altivec(&tsk->thread);
+ load_vr_state(&tsk->thread.vr_state);
regs->msr |= MSR_VEC;
}
-#endif
return err;
}
/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */
std r3, STK_PARAM(R3)(r1)
+ std r4, STK_PARAM(R4)(r1)
SAVE_NVGPRS(r1)
/* We need to setup MSR for VSX register save instructions. */
mtmsrd r15
std r14, TM_FRAME_L0(r1)
- /* Stash the stack pointer away for use after reclaim */
- std r1, PACAR1(r13)
-
- /* ******************** FPR/VR/VSRs ************
- * Before reclaiming, capture the current/transactional FPR/VR
- * versions /if used/.
- *
- * (If VSX used, FP and VMX are implied. Or, we don't need to look
- * at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
- *
- * We're passed the thread's MSR as parameter 2.
- *
- * We enabled VEC/FP/VSX in the msr above, so we can execute these
- * instructions!
- */
- andis. r0, r4, MSR_VEC@h
- beq dont_backup_vec
-
- addi r7, r3, THREAD_TRANSACT_VRSTATE
- SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
- mfvscr v0
- li r6, VRSTATE_VSCR
- stvx v0, r7, r6
-dont_backup_vec:
- mfspr r0, SPRN_VRSAVE
- std r0, THREAD_TRANSACT_VRSAVE(r3)
-
- andi. r0, r4, MSR_FP
- beq dont_backup_fp
-
- addi r7, r3, THREAD_TRANSACT_FPSTATE
- SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
-
- mffs fr0
- stfd fr0,FPSTATE_FPSCR(r7)
-
-dont_backup_fp:
/* Do sanity check on MSR to make sure we are suspended */
li r7, (MSR_TS_S)@higher
srdi r6, r14, 32
1: tdeqi r6, 0
EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
+ /* Stash the stack pointer away for use after reclaim */
+ std r1, PACAR1(r13)
+
/* Clear MSR RI since we are about to change r1, EE is already off. */
li r4, 0
mtmsrd r4, 1
* MSR.
*/
+
+ /* ******************** FPR/VR/VSRs ************
+ * After reclaiming, capture the checkpointed FPRs/VRs /if used/.
+ *
+ * (If VSX used, FP and VMX are implied. Or, we don't need to look
+ * at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
+ *
+ * We're passed the thread's MSR as the second parameter
+ *
+ * We enabled VEC/FP/VSX in the msr above, so we can execute these
+ * instructions!
+ */
+ ld r4, STK_PARAM(R4)(r1) /* Second parameter, MSR * */
+ mr r3, r12
+ andis. r0, r4, MSR_VEC@h
+ beq dont_backup_vec
+
+ addi r7, r3, THREAD_TRANSACT_VRSTATE
+ SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
+ mfvscr v0
+ li r6, VRSTATE_VSCR
+ stvx v0, r7, r6
+dont_backup_vec:
+ mfspr r0, SPRN_VRSAVE
+ std r0, THREAD_TRANSACT_VRSAVE(r3)
+
+ andi. r0, r4, MSR_FP
+ beq dont_backup_fp
+
+ addi r7, r3, THREAD_TRANSACT_FPSTATE
+ SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
+
+ mffs fr0
+ stfd fr0,FPSTATE_FPSCR(r7)
+
+dont_backup_fp:
+
/* TM regs, incl TEXASR -- these live in thread_struct. Note they've
* been updated by the treclaim, to explain to userland the failure
* cause (aborted).
/* Restore original MSR/IRQ state & clear TM mode */
ld r14, TM_FRAME_L0(r1) /* Orig MSR */
+
li r15, 0
rldimi r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
mtmsrd r14
mtmsr r5
#ifdef CONFIG_ALTIVEC
- /* FP and VEC registers: These are recheckpointed from thread.fpr[]
- * and thread.vr[] respectively. The thread.transact_fpr[] version
- * is more modern, and will be loaded subsequently by any FPUnavailable
- * trap.
+ /*
+ * FP and VEC registers: These are recheckpointed from
+ * thread.ckfp_state and thread.ckvr_state respectively. The
+ * thread.fp_state[] version holds the 'live' (transactional)
+ * and will be loaded subsequently by any FPUnavailable trap.
*/
andis. r0, r4, MSR_VEC@h
beq dont_restore_vec
- addi r8, r3, THREAD_VRSTATE
+ addi r8, r3, THREAD_TRANSACT_VRSTATE
li r5, VRSTATE_VSCR
lvx v0, r8, r5
mtvscr v0
REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */
dont_restore_vec:
- ld r5, THREAD_VRSAVE(r3)
+ ld r5, THREAD_TRANSACT_VRSAVE(r3)
mtspr SPRN_VRSAVE, r5
#endif
andi. r0, r4, MSR_FP
beq dont_restore_fp
- addi r8, r3, THREAD_FPSTATE
+ addi r8, r3, THREAD_TRANSACT_FPSTATE
lfd fr0, FPSTATE_FPSCR(r8)
MTFSF_L(fr0)
REST_32FPRS_VSRS(0, R4, R8)
projects / platform / kernel / linux-exynos.git / commitdiff