2 * Copyright (C) 1994 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * General FPU state handling cleanups
6 * Gareth Hughes <gareth@valinux.com>, May 2000
8 #include <asm/fpu/internal.h>
9 #include <asm/fpu/regset.h>
10 #include <asm/fpu/signal.h>
11 #include <asm/fpu/types.h>
12 #include <asm/traps.h>
14 #include <linux/hardirq.h>
16 #define CREATE_TRACE_POINTS
17 #include <asm/trace/fpu.h>
20 * Represents the initial FPU state. It's mostly (but not completely) zeroes,
21 * depending on the FPU hardware format:
23 union fpregs_state init_fpstate __read_mostly;
26 * Track whether the kernel is using the FPU state
31 * - by IRQ context code to potentially use the FPU
34 * - to debug kernel_fpu_begin()/end() correctness
36 static DEFINE_PER_CPU(bool, in_kernel_fpu);
39 * Track which context is using the FPU on the CPU:
41 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
43 static void kernel_fpu_disable(void)
45 WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
46 this_cpu_write(in_kernel_fpu, true);
49 static void kernel_fpu_enable(void)
51 WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
52 this_cpu_write(in_kernel_fpu, false);
55 static bool kernel_fpu_disabled(void)
57 return this_cpu_read(in_kernel_fpu);
60 static bool interrupted_kernel_fpu_idle(void)
62 return !kernel_fpu_disabled();
66 * Were we in user mode (or vm86 mode) when we were
69 * Doing kernel_fpu_begin/end() is ok if we are running
70 * in an interrupt context from user mode - we'll just
71 * save the FPU state as required.
73 static bool interrupted_user_mode(void)
75 struct pt_regs *regs = get_irq_regs();
76 return regs && user_mode(regs);
80 * Can we use the FPU in kernel mode with the
81 * whole "kernel_fpu_begin/end()" sequence?
83 * It's always ok in process context (ie "not interrupt")
84 * but it is sometimes ok even from an irq.
86 bool irq_fpu_usable(void)
88 return !in_interrupt() ||
89 interrupted_user_mode() ||
90 interrupted_kernel_fpu_idle();
92 EXPORT_SYMBOL(irq_fpu_usable);
94 void __kernel_fpu_begin(void)
96 struct fpu *fpu = ¤t->thread.fpu;
98 WARN_ON_FPU(!irq_fpu_usable());
100 kernel_fpu_disable();
102 if (fpu->fpregs_active) {
104 * Ignore return value -- we don't care if reg state
107 copy_fpregs_to_fpstate(fpu);
109 this_cpu_write(fpu_fpregs_owner_ctx, NULL);
112 EXPORT_SYMBOL(__kernel_fpu_begin);
114 void __kernel_fpu_end(void)
116 struct fpu *fpu = ¤t->thread.fpu;
118 if (fpu->fpregs_active)
119 copy_kernel_to_fpregs(&fpu->state);
123 EXPORT_SYMBOL(__kernel_fpu_end);
125 void kernel_fpu_begin(void)
128 __kernel_fpu_begin();
130 EXPORT_SYMBOL_GPL(kernel_fpu_begin);
132 void kernel_fpu_end(void)
137 EXPORT_SYMBOL_GPL(kernel_fpu_end);
140 * CR0::TS save/restore functions:
142 int irq_ts_save(void)
145 * If in process context and not atomic, we can take a spurious DNA fault.
146 * Otherwise, doing clts() in process context requires disabling preemption
147 * or some heavy lifting like kernel_fpu_begin()
152 if (read_cr0() & X86_CR0_TS) {
159 EXPORT_SYMBOL_GPL(irq_ts_save);
161 void irq_ts_restore(int TS_state)
166 EXPORT_SYMBOL_GPL(irq_ts_restore);
169 * Save the FPU state (mark it for reload if necessary):
171 * This only ever gets called for the current task.
173 void fpu__save(struct fpu *fpu)
175 WARN_ON_FPU(fpu != ¤t->thread.fpu);
178 trace_x86_fpu_before_save(fpu);
179 if (fpu->fpregs_active) {
180 if (!copy_fpregs_to_fpstate(fpu)) {
181 copy_kernel_to_fpregs(&fpu->state);
184 trace_x86_fpu_after_save(fpu);
187 EXPORT_SYMBOL_GPL(fpu__save);
190 * Legacy x87 fpstate state init:
192 static inline void fpstate_init_fstate(struct fregs_state *fp)
194 fp->cwd = 0xffff037fu;
195 fp->swd = 0xffff0000u;
196 fp->twd = 0xffffffffu;
197 fp->fos = 0xffff0000u;
200 void fpstate_init(union fpregs_state *state)
202 if (!static_cpu_has(X86_FEATURE_FPU)) {
203 fpstate_init_soft(&state->soft);
207 memset(state, 0, fpu_kernel_xstate_size);
210 * XRSTORS requires that this bit is set in xcomp_bv, or
211 * it will #GP. Make sure it is replaced after the memset().
213 if (static_cpu_has(X86_FEATURE_XSAVES))
214 state->xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT;
216 if (static_cpu_has(X86_FEATURE_FXSR))
217 fpstate_init_fxstate(&state->fxsave);
219 fpstate_init_fstate(&state->fsave);
221 EXPORT_SYMBOL_GPL(fpstate_init);
223 int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
225 dst_fpu->fpregs_active = 0;
226 dst_fpu->last_cpu = -1;
228 if (!src_fpu->fpstate_active || !static_cpu_has(X86_FEATURE_FPU))
231 WARN_ON_FPU(src_fpu != ¤t->thread.fpu);
234 * Don't let 'init optimized' areas of the XSAVE area
235 * leak into the child task:
237 memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size);
240 * Save current FPU registers directly into the child
241 * FPU context, without any memory-to-memory copying.
242 * In lazy mode, if the FPU context isn't loaded into
243 * fpregs, CR0.TS will be set and do_device_not_available
244 * will load the FPU context.
246 * We have to do all this with preemption disabled,
247 * mostly because of the FNSAVE case, because in that
248 * case we must not allow preemption in the window
249 * between the FNSAVE and us marking the context lazy.
251 * It shouldn't be an issue as even FNSAVE is plenty
252 * fast in terms of critical section length.
255 if (!copy_fpregs_to_fpstate(dst_fpu)) {
256 memcpy(&src_fpu->state, &dst_fpu->state,
257 fpu_kernel_xstate_size);
259 copy_kernel_to_fpregs(&src_fpu->state);
263 trace_x86_fpu_copy_src(src_fpu);
264 trace_x86_fpu_copy_dst(dst_fpu);
270 * Activate the current task's in-memory FPU context,
271 * if it has not been used before:
273 void fpu__activate_curr(struct fpu *fpu)
275 WARN_ON_FPU(fpu != ¤t->thread.fpu);
277 if (!fpu->fpstate_active) {
278 fpstate_init(&fpu->state);
279 trace_x86_fpu_init_state(fpu);
281 trace_x86_fpu_activate_state(fpu);
282 /* Safe to do for the current task: */
283 fpu->fpstate_active = 1;
286 EXPORT_SYMBOL_GPL(fpu__activate_curr);
289 * This function must be called before we read a task's fpstate.
291 * If the task has not used the FPU before then initialize its
294 * If the task has used the FPU before then save it.
296 void fpu__activate_fpstate_read(struct fpu *fpu)
299 * If fpregs are active (in the current CPU), then
300 * copy them to the fpstate:
302 if (fpu->fpregs_active) {
305 if (!fpu->fpstate_active) {
306 fpstate_init(&fpu->state);
307 trace_x86_fpu_init_state(fpu);
309 trace_x86_fpu_activate_state(fpu);
310 /* Safe to do for current and for stopped child tasks: */
311 fpu->fpstate_active = 1;
317 * This function must be called before we write a task's fpstate.
319 * If the task has used the FPU before then unlazy it.
320 * If the task has not used the FPU before then initialize its fpstate.
322 * After this function call, after registers in the fpstate are
323 * modified and the child task has woken up, the child task will
324 * restore the modified FPU state from the modified context. If we
325 * didn't clear its lazy status here then the lazy in-registers
326 * state pending on its former CPU could be restored, corrupting
329 void fpu__activate_fpstate_write(struct fpu *fpu)
332 * Only stopped child tasks can be used to modify the FPU
333 * state in the fpstate buffer:
335 WARN_ON_FPU(fpu == ¤t->thread.fpu);
337 if (fpu->fpstate_active) {
338 /* Invalidate any lazy state: */
341 fpstate_init(&fpu->state);
342 trace_x86_fpu_init_state(fpu);
344 trace_x86_fpu_activate_state(fpu);
345 /* Safe to do for stopped child tasks: */
346 fpu->fpstate_active = 1;
351 * This function must be called before we write the current
354 * This call gets the current FPU register state and moves
355 * it in to the 'fpstate'. Preemption is disabled so that
356 * no writes to the 'fpstate' can occur from context
359 * Must be followed by a fpu__current_fpstate_write_end().
361 void fpu__current_fpstate_write_begin(void)
363 struct fpu *fpu = ¤t->thread.fpu;
366 * Ensure that the context-switching code does not write
367 * over the fpstate while we are doing our update.
372 * Move the fpregs in to the fpu's 'fpstate'.
374 fpu__activate_fpstate_read(fpu);
377 * The caller is about to write to 'fpu'. Ensure that no
378 * CPU thinks that its fpregs match the fpstate. This
379 * ensures we will not be lazy and skip a XRSTOR in the
386 * This function must be paired with fpu__current_fpstate_write_begin()
388 * This will ensure that the modified fpstate gets placed back in
389 * the fpregs if necessary.
391 * Note: This function may be called whether or not an _actual_
392 * write to the fpstate occurred.
394 void fpu__current_fpstate_write_end(void)
396 struct fpu *fpu = ¤t->thread.fpu;
399 * 'fpu' now has an updated copy of the state, but the
400 * registers may still be out of date. Update them with
401 * an XRSTOR if they are active.
404 copy_kernel_to_fpregs(&fpu->state);
407 * Our update is done and the fpregs/fpstate are in sync
408 * if necessary. Context switches can happen again.
414 * 'fpu__restore()' is called to copy FPU registers from
415 * the FPU fpstate to the live hw registers and to activate
416 * access to the hardware registers, so that FPU instructions
417 * can be used afterwards.
419 * Must be called with kernel preemption disabled (for example
420 * with local interrupts disabled, as it is in the case of
421 * do_device_not_available()).
423 void fpu__restore(struct fpu *fpu)
425 fpu__activate_curr(fpu);
427 /* Avoid __kernel_fpu_begin() right after fpregs_activate() */
428 kernel_fpu_disable();
429 trace_x86_fpu_before_restore(fpu);
430 fpregs_activate(fpu);
431 copy_kernel_to_fpregs(&fpu->state);
432 trace_x86_fpu_after_restore(fpu);
435 EXPORT_SYMBOL_GPL(fpu__restore);
438 * Drops current FPU state: deactivates the fpregs and
439 * the fpstate. NOTE: it still leaves previous contents
440 * in the fpregs in the eager-FPU case.
442 * This function can be used in cases where we know that
443 * a state-restore is coming: either an explicit one,
446 void fpu__drop(struct fpu *fpu)
450 if (fpu->fpregs_active) {
451 /* Ignore delayed exceptions from user space */
452 asm volatile("1: fwait\n"
454 _ASM_EXTABLE(1b, 2b));
455 fpregs_deactivate(fpu);
458 fpu->fpstate_active = 0;
460 trace_x86_fpu_dropped(fpu);
466 * Clear FPU registers by setting them up from
469 static inline void copy_init_fpstate_to_fpregs(void)
472 copy_kernel_to_xregs(&init_fpstate.xsave, -1);
473 else if (static_cpu_has(X86_FEATURE_FXSR))
474 copy_kernel_to_fxregs(&init_fpstate.fxsave);
476 copy_kernel_to_fregs(&init_fpstate.fsave);
480 * Clear the FPU state back to init state.
482 * Called by sys_execve(), by the signal handler code and by various
485 void fpu__clear(struct fpu *fpu)
487 WARN_ON_FPU(fpu != ¤t->thread.fpu); /* Almost certainly an anomaly */
489 if (!static_cpu_has(X86_FEATURE_FPU)) {
490 /* FPU state will be reallocated lazily at the first use. */
493 if (!fpu->fpstate_active) {
494 fpu__activate_curr(fpu);
497 copy_init_fpstate_to_fpregs();
502 * x87 math exception handling:
505 int fpu__exception_code(struct fpu *fpu, int trap_nr)
509 if (trap_nr == X86_TRAP_MF) {
510 unsigned short cwd, swd;
512 * (~cwd & swd) will mask out exceptions that are not set to unmasked
513 * status. 0x3f is the exception bits in these regs, 0x200 is the
514 * C1 reg you need in case of a stack fault, 0x040 is the stack
515 * fault bit. We should only be taking one exception at a time,
516 * so if this combination doesn't produce any single exception,
517 * then we have a bad program that isn't synchronizing its FPU usage
518 * and it will suffer the consequences since we won't be able to
519 * fully reproduce the context of the exception.
521 if (boot_cpu_has(X86_FEATURE_FXSR)) {
522 cwd = fpu->state.fxsave.cwd;
523 swd = fpu->state.fxsave.swd;
525 cwd = (unsigned short)fpu->state.fsave.cwd;
526 swd = (unsigned short)fpu->state.fsave.swd;
532 * The SIMD FPU exceptions are handled a little differently, as there
533 * is only a single status/control register. Thus, to determine which
534 * unmasked exception was caught we must mask the exception mask bits
535 * at 0x1f80, and then use these to mask the exception bits at 0x3f.
537 unsigned short mxcsr = MXCSR_DEFAULT;
539 if (boot_cpu_has(X86_FEATURE_XMM))
540 mxcsr = fpu->state.fxsave.mxcsr;
542 err = ~(mxcsr >> 7) & mxcsr;
545 if (err & 0x001) { /* Invalid op */
547 * swd & 0x240 == 0x040: Stack Underflow
548 * swd & 0x240 == 0x240: Stack Overflow
549 * User must clear the SF bit (0x40) if set
552 } else if (err & 0x004) { /* Divide by Zero */
554 } else if (err & 0x008) { /* Overflow */
556 } else if (err & 0x012) { /* Denormal, Underflow */
558 } else if (err & 0x020) { /* Precision */
563 * If we're using IRQ 13, or supposedly even some trap
564 * X86_TRAP_MF implementations, it's possible
565 * we get a spurious trap, which is not an error.