arch/x86/include/asm/fpu-internal.h

   1 /*
   2  * Copyright (C) 1994 Linus Torvalds
   3  *
   4  * Pentium III FXSR, SSE support
   5  * General FPU state handling cleanups
   6  *      Gareth Hughes <gareth@valinux.com>, May 2000
   7  * x86-64 work by Andi Kleen 2002
   8  */
   9
  10 #ifndef _FPU_INTERNAL_H
  11 #define _FPU_INTERNAL_H
  12
  13 #include <linux/kernel_stat.h>
  14 #include <linux/regset.h>
  15 #include <linux/compat.h>
  16 #include <linux/slab.h>
  17 #include <asm/asm.h>
  18 #include <asm/cpufeature.h>
  19 #include <asm/processor.h>
  20 #include <asm/sigcontext.h>
  21 #include <asm/user.h>
  22 #include <asm/uaccess.h>
  23 #include <asm/xsave.h>
  24
  25 extern unsigned int sig_xstate_size;
  26 extern void fpu_init(void);
  27
  28 DECLARE_PER_CPU(struct task_struct *, fpu_owner_task);
  29
  30 extern user_regset_active_fn fpregs_active, xfpregs_active;
  31 extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
  32                                 xstateregs_get;
  33 extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
  34                                  xstateregs_set;
  35
  36 /*
  37  * xstateregs_active == fpregs_active. Please refer to the comment
  38  * at the definition of fpregs_active.
  39  */
  40 #define xstateregs_active       fpregs_active
  41
  42 extern struct _fpx_sw_bytes fx_sw_reserved;
  43 #ifdef CONFIG_IA32_EMULATION
  44 extern unsigned int sig_xstate_ia32_size;
  45 extern struct _fpx_sw_bytes fx_sw_reserved_ia32;
  46 struct _fpstate_ia32;
  47 struct _xstate_ia32;
  48 extern int save_i387_xstate_ia32(void __user *buf);
  49 extern int restore_i387_xstate_ia32(void __user *buf);
  50 #endif
  51
  52 #ifdef CONFIG_MATH_EMULATION
  53 extern void finit_soft_fpu(struct i387_soft_struct *soft);
  54 #else
  55 static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
  56 #endif
  57
  58 static inline int is_ia32_compat_frame(void)
  59 {
  60         return config_enabled(CONFIG_IA32_EMULATION) &&
  61                test_thread_flag(TIF_IA32);
  62 }
  63
  64 static inline int is_ia32_frame(void)
  65 {
  66         return config_enabled(CONFIG_X86_32) || is_ia32_compat_frame();
  67 }
  68
  69 static inline int is_x32_frame(void)
  70 {
  71         return config_enabled(CONFIG_X86_X32_ABI) && test_thread_flag(TIF_X32);
  72 }
  73
  74 #define X87_FSW_ES (1 << 7)     /* Exception Summary */
  75
  76 static __always_inline __pure bool use_xsaveopt(void)
  77 {
  78         return static_cpu_has(X86_FEATURE_XSAVEOPT);
  79 }
  80
  81 static __always_inline __pure bool use_xsave(void)
  82 {
  83         return static_cpu_has(X86_FEATURE_XSAVE);
  84 }
  85
  86 static __always_inline __pure bool use_fxsr(void)
  87 {
  88         return static_cpu_has(X86_FEATURE_FXSR);
  89 }
  90
  91 extern void __sanitize_i387_state(struct task_struct *);
  92
  93 static inline void sanitize_i387_state(struct task_struct *tsk)
  94 {
  95         if (!use_xsaveopt())
  96                 return;
  97         __sanitize_i387_state(tsk);
  98 }
  99
 100 #define check_insn(insn, output, input...)                              \
 101 ({                                                                      \
 102         int err;                                                        \
 103         asm volatile("1:" #insn "\n\t"                                  \
 104                      "2:\n"                                             \
 105                      ".section .fixup,\"ax\"\n"                         \
 106                      "3:  movl $-1,%[err]\n"                            \
 107                      "    jmp  2b\n"                                    \
 108                      ".previous\n"                                      \
 109                      _ASM_EXTABLE(1b, 3b)                               \
 110                      : [err] "=r" (err), output                         \
 111                      : "0"(0), input);                                  \
 112         err;                                                            \
 113 })
 114
 115 static inline int fsave_user(struct i387_fsave_struct __user *fx)
 116 {
 117         return check_insn(fnsave %[fx]; fwait,  [fx] "=m" (*fx), "m" (*fx));
 118 }
 119
 120 static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
 121 {
 122         int err;
 123
 124         /*
 125          * Clear the bytes not touched by the fxsave and reserved
 126          * for the SW usage.
 127          */
 128         err = __clear_user(&fx->sw_reserved,
 129                            sizeof(struct _fpx_sw_bytes));
 130         if (unlikely(err))
 131                 return -EFAULT;
 132
 133         if (config_enabled(CONFIG_X86_32))
 134                 return check_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
 135         else if (config_enabled(CONFIG_AS_FXSAVEQ))
 136                 return check_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));
 137
 138         /* See comment in fpu_fxsave() below. */
 139         return check_insn(rex64/fxsave (%[fx]), "=m" (*fx), [fx] "R" (fx));
 140 }
 141
 142 static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
 143 {
 144         if (config_enabled(CONFIG_X86_32))
 145                 return check_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 146         else if (config_enabled(CONFIG_AS_FXSAVEQ))
 147                 return check_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
 148
 149         /* See comment in fpu_fxsave() below. */
 150         return check_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx),
 151                           "m" (*fx));
 152 }
 153
 154 static inline int frstor_checking(struct i387_fsave_struct *fx)
 155 {
 156         return check_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 157 }
 158
 159 static inline void fpu_fxsave(struct fpu *fpu)
 160 {
 161         if (config_enabled(CONFIG_X86_32))
 162                 asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state->fxsave));
 163         else if (config_enabled(CONFIG_AS_FXSAVEQ))
 164                 asm volatile("fxsaveq %0" : "=m" (fpu->state->fxsave));
 165         else {
 166                 /* Using "rex64; fxsave %0" is broken because, if the memory
 167                  * operand uses any extended registers for addressing, a second
 168                  * REX prefix will be generated (to the assembler, rex64
 169                  * followed by semicolon is a separate instruction), and hence
 170                  * the 64-bitness is lost.
 171                  *
 172                  * Using "fxsaveq %0" would be the ideal choice, but is only
 173                  * supported starting with gas 2.16.
 174                  *
 175                  * Using, as a workaround, the properly prefixed form below
 176                  * isn't accepted by any binutils version so far released,
 177                  * complaining that the same type of prefix is used twice if
 178                  * an extended register is needed for addressing (fix submitted
 179                  * to mainline 2005-11-21).
 180                  *
 181                  *  asm volatile("rex64/fxsave %0" : "=m" (fpu->state->fxsave));
 182                  *
 183                  * This, however, we can work around by forcing the compiler to
 184                  * select an addressing mode that doesn't require extended
 185                  * registers.
 186                  */
 187                 asm volatile( "rex64/fxsave (%[fx])"
 188                              : "=m" (fpu->state->fxsave)
 189                              : [fx] "R" (&fpu->state->fxsave));
 190         }
 191 }
 192 #ifdef CONFIG_X86_64
 193
 194 int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 195                         compat_sigset_t *set, struct pt_regs *regs);
 196 int ia32_setup_frame(int sig, struct k_sigaction *ka,
 197                      compat_sigset_t *set, struct pt_regs *regs);
 198
 199 #else  /* CONFIG_X86_32 */
 200
 201 #define ia32_setup_frame        __setup_frame
 202 #define ia32_setup_rt_frame     __setup_rt_frame
 203
 204 #endif  /* CONFIG_X86_64 */
 205
 206 /*
 207  * These must be called with preempt disabled. Returns
 208  * 'true' if the FPU state is still intact.
 209  */
 210 static inline int fpu_save_init(struct fpu *fpu)
 211 {
 212         if (use_xsave()) {
 213                 fpu_xsave(fpu);
 214
 215                 /*
 216                  * xsave header may indicate the init state of the FP.
 217                  */
 218                 if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP))
 219                         return 1;
 220         } else if (use_fxsr()) {
 221                 fpu_fxsave(fpu);
 222         } else {
 223                 asm volatile("fnsave %[fx]; fwait"
 224                              : [fx] "=m" (fpu->state->fsave));
 225                 return 0;
 226         }
 227
 228         /*
 229          * If exceptions are pending, we need to clear them so
 230          * that we don't randomly get exceptions later.
 231          *
 232          * FIXME! Is this perhaps only true for the old-style
 233          * irq13 case? Maybe we could leave the x87 state
 234          * intact otherwise?
 235          */
 236         if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) {
 237                 asm volatile("fnclex");
 238                 return 0;
 239         }
 240         return 1;
 241 }
 242
 243 static inline int __save_init_fpu(struct task_struct *tsk)
 244 {
 245         return fpu_save_init(&tsk->thread.fpu);
 246 }
 247
 248 static inline int fpu_restore_checking(struct fpu *fpu)
 249 {
 250         if (use_xsave())
 251                 return fpu_xrstor_checking(&fpu->state->xsave);
 252         else if (use_fxsr())
 253                 return fxrstor_checking(&fpu->state->fxsave);
 254         else
 255                 return frstor_checking(&fpu->state->fsave);
 256 }
 257
 258 static inline int restore_fpu_checking(struct task_struct *tsk)
 259 {
 260         /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
 261            is pending.  Clear the x87 state here by setting it to fixed
 262            values. "m" is a random variable that should be in L1 */
 263         alternative_input(
 264                 ASM_NOP8 ASM_NOP2,
 265                 "emms\n\t"              /* clear stack tags */
 266                 "fildl %P[addr]",       /* set F?P to defined value */
 267                 X86_FEATURE_FXSAVE_LEAK,
 268                 [addr] "m" (tsk->thread.fpu.has_fpu));
 269
 270         return fpu_restore_checking(&tsk->thread.fpu);
 271 }
 272
 273 /*
 274  * Software FPU state helpers. Careful: these need to
 275  * be preemption protection *and* they need to be
 276  * properly paired with the CR0.TS changes!
 277  */
 278 static inline int __thread_has_fpu(struct task_struct *tsk)
 279 {
 280         return tsk->thread.fpu.has_fpu;
 281 }
 282
 283 /* Must be paired with an 'stts' after! */
 284 static inline void __thread_clear_has_fpu(struct task_struct *tsk)
 285 {
 286         tsk->thread.fpu.has_fpu = 0;
 287         this_cpu_write(fpu_owner_task, NULL);
 288 }
 289
 290 /* Must be paired with a 'clts' before! */
 291 static inline void __thread_set_has_fpu(struct task_struct *tsk)
 292 {
 293         tsk->thread.fpu.has_fpu = 1;
 294         this_cpu_write(fpu_owner_task, tsk);
 295 }
 296
 297 /*
 298  * Encapsulate the CR0.TS handling together with the
 299  * software flag.
 300  *
 301  * These generally need preemption protection to work,
 302  * do try to avoid using these on their own.
 303  */
 304 static inline void __thread_fpu_end(struct task_struct *tsk)
 305 {
 306         __thread_clear_has_fpu(tsk);
 307         stts();
 308 }
 309
 310 static inline void __thread_fpu_begin(struct task_struct *tsk)
 311 {
 312         clts();
 313         __thread_set_has_fpu(tsk);
 314 }
 315
 316 /*
 317  * FPU state switching for scheduling.
 318  *
 319  * This is a two-stage process:
 320  *
 321  *  - switch_fpu_prepare() saves the old state and
 322  *    sets the new state of the CR0.TS bit. This is
 323  *    done within the context of the old process.
 324  *
 325  *  - switch_fpu_finish() restores the new state as
 326  *    necessary.
 327  */
 328 typedef struct { int preload; } fpu_switch_t;
 329
 330 /*
 331  * FIXME! We could do a totally lazy restore, but we need to
 332  * add a per-cpu "this was the task that last touched the FPU
 333  * on this CPU" variable, and the task needs to have a "I last
 334  * touched the FPU on this CPU" and check them.
 335  *
 336  * We don't do that yet, so "fpu_lazy_restore()" always returns
 337  * false, but some day..
 338  */
 339 static inline int fpu_lazy_restore(struct task_struct *new, unsigned int cpu)
 340 {
 341         return new == this_cpu_read_stable(fpu_owner_task) &&
 342                 cpu == new->thread.fpu.last_cpu;
 343 }
 344
 345 static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct task_struct *new, int cpu)
 346 {
 347         fpu_switch_t fpu;
 348
 349         fpu.preload = tsk_used_math(new) && new->fpu_counter > 5;
 350         if (__thread_has_fpu(old)) {
 351                 if (!__save_init_fpu(old))
 352                         cpu = ~0;
 353                 old->thread.fpu.last_cpu = cpu;
 354                 old->thread.fpu.has_fpu = 0;    /* But leave fpu_owner_task! */
 355
 356                 /* Don't change CR0.TS if we just switch! */
 357                 if (fpu.preload) {
 358                         new->fpu_counter++;
 359                         __thread_set_has_fpu(new);
 360                         prefetch(new->thread.fpu.state);
 361                 } else
 362                         stts();
 363         } else {
 364                 old->fpu_counter = 0;
 365                 old->thread.fpu.last_cpu = ~0;
 366                 if (fpu.preload) {
 367                         new->fpu_counter++;
 368                         if (fpu_lazy_restore(new, cpu))
 369                                 fpu.preload = 0;
 370                         else
 371                                 prefetch(new->thread.fpu.state);
 372                         __thread_fpu_begin(new);
 373                 }
 374         }
 375         return fpu;
 376 }
 377
 378 /*
 379  * By the time this gets called, we've already cleared CR0.TS and
 380  * given the process the FPU if we are going to preload the FPU
 381  * state - all we need to do is to conditionally restore the register
 382  * state itself.
 383  */
 384 static inline void switch_fpu_finish(struct task_struct *new, fpu_switch_t fpu)
 385 {
 386         if (fpu.preload) {
 387                 if (unlikely(restore_fpu_checking(new)))
 388                         __thread_fpu_end(new);
 389         }
 390 }
 391
 392 /*
 393  * Signal frame handlers...
 394  */
 395 extern int save_i387_xstate(void __user *buf);
 396 extern int restore_i387_xstate(void __user *buf);
 397
 398 static inline void __clear_fpu(struct task_struct *tsk)
 399 {
 400         if (__thread_has_fpu(tsk)) {
 401                 /* Ignore delayed exceptions from user space */
 402                 asm volatile("1: fwait\n"
 403                              "2:\n"
 404                              _ASM_EXTABLE(1b, 2b));
 405                 __thread_fpu_end(tsk);
 406         }
 407 }
 408
 409 /*
 410  * The actual user_fpu_begin/end() functions
 411  * need to be preemption-safe.
 412  *
 413  * NOTE! user_fpu_end() must be used only after you
 414  * have saved the FP state, and user_fpu_begin() must
 415  * be used only immediately before restoring it.
 416  * These functions do not do any save/restore on
 417  * their own.
 418  */
 419 static inline void user_fpu_end(void)
 420 {
 421         preempt_disable();
 422         __thread_fpu_end(current);
 423         preempt_enable();
 424 }
 425
 426 static inline void user_fpu_begin(void)
 427 {
 428         preempt_disable();
 429         if (!user_has_fpu())
 430                 __thread_fpu_begin(current);
 431         preempt_enable();
 432 }
 433
 434 /*
 435  * These disable preemption on their own and are safe
 436  */
 437 static inline void save_init_fpu(struct task_struct *tsk)
 438 {
 439         WARN_ON_ONCE(!__thread_has_fpu(tsk));
 440         preempt_disable();
 441         __save_init_fpu(tsk);
 442         __thread_fpu_end(tsk);
 443         preempt_enable();
 444 }
 445
 446 static inline void clear_fpu(struct task_struct *tsk)
 447 {
 448         preempt_disable();
 449         __clear_fpu(tsk);
 450         preempt_enable();
 451 }
 452
 453 /*
 454  * i387 state interaction
 455  */
 456 static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
 457 {
 458         if (cpu_has_fxsr) {
 459                 return tsk->thread.fpu.state->fxsave.cwd;
 460         } else {
 461                 return (unsigned short)tsk->thread.fpu.state->fsave.cwd;
 462         }
 463 }
 464
 465 static inline unsigned short get_fpu_swd(struct task_struct *tsk)
 466 {
 467         if (cpu_has_fxsr) {
 468                 return tsk->thread.fpu.state->fxsave.swd;
 469         } else {
 470                 return (unsigned short)tsk->thread.fpu.state->fsave.swd;
 471         }
 472 }
 473
 474 static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
 475 {
 476         if (cpu_has_xmm) {
 477                 return tsk->thread.fpu.state->fxsave.mxcsr;
 478         } else {
 479                 return MXCSR_DEFAULT;
 480         }
 481 }
 482
 483 static bool fpu_allocated(struct fpu *fpu)
 484 {
 485         return fpu->state != NULL;
 486 }
 487
 488 static inline int fpu_alloc(struct fpu *fpu)
 489 {
 490         if (fpu_allocated(fpu))
 491                 return 0;
 492         fpu->state = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL);
 493         if (!fpu->state)
 494                 return -ENOMEM;
 495         WARN_ON((unsigned long)fpu->state & 15);
 496         return 0;
 497 }
 498
 499 static inline void fpu_free(struct fpu *fpu)
 500 {
 501         if (fpu->state) {
 502                 kmem_cache_free(task_xstate_cachep, fpu->state);
 503                 fpu->state = NULL;
 504         }
 505 }
 506
 507 static inline void fpu_copy(struct fpu *dst, struct fpu *src)
 508 {
 509         memcpy(dst->state, src->state, xstate_size);
 510 }
 511
 512 extern void fpu_finit(struct fpu *fpu);
 513
 514 #endif