1 #ifndef _ASM_X86_PARAVIRT_H
2 #define _ASM_X86_PARAVIRT_H
3 /* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
7 #include <asm/pgtable_types.h>
10 #include <asm/paravirt_types.h>
13 #include <linux/types.h>
14 #include <linux/cpumask.h>
16 static inline int paravirt_enabled(void)
18 return pv_info.paravirt_enabled;
21 static inline void load_sp0(struct tss_struct *tss,
22 struct thread_struct *thread)
24 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
27 /* The paravirtualized CPUID instruction. */
28 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
29 unsigned int *ecx, unsigned int *edx)
31 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
35 * These special macros can be used to get or set a debugging register
37 static inline unsigned long paravirt_get_debugreg(int reg)
39 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
41 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
42 static inline void set_debugreg(unsigned long val, int reg)
44 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
47 static inline void clts(void)
49 PVOP_VCALL0(pv_cpu_ops.clts);
52 static inline unsigned long read_cr0(void)
54 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
57 static inline void write_cr0(unsigned long x)
59 PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
62 static inline unsigned long read_cr2(void)
64 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
67 static inline void write_cr2(unsigned long x)
69 PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
72 static inline unsigned long read_cr3(void)
74 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
77 static inline void write_cr3(unsigned long x)
79 PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
82 static inline unsigned long read_cr4(void)
84 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
86 static inline unsigned long read_cr4_safe(void)
88 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
91 static inline void write_cr4(unsigned long x)
93 PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
97 static inline unsigned long read_cr8(void)
99 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
102 static inline void write_cr8(unsigned long x)
104 PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
108 static inline void raw_safe_halt(void)
110 PVOP_VCALL0(pv_irq_ops.safe_halt);
113 static inline void halt(void)
115 PVOP_VCALL0(pv_irq_ops.safe_halt);
118 static inline void wbinvd(void)
120 PVOP_VCALL0(pv_cpu_ops.wbinvd);
123 #define get_kernel_rpl() (pv_info.kernel_rpl)
125 static inline u64 paravirt_read_msr(unsigned msr, int *err)
127 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
130 static inline int paravirt_rdmsr_regs(u32 *regs)
132 return PVOP_CALL1(int, pv_cpu_ops.rdmsr_regs, regs);
135 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
137 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
140 static inline int paravirt_wrmsr_regs(u32 *regs)
142 return PVOP_CALL1(int, pv_cpu_ops.wrmsr_regs, regs);
145 /* These should all do BUG_ON(_err), but our headers are too tangled. */
146 #define rdmsr(msr, val1, val2) \
149 u64 _l = paravirt_read_msr(msr, &_err); \
154 #define wrmsr(msr, val1, val2) \
156 paravirt_write_msr(msr, val1, val2); \
159 #define rdmsrl(msr, val) \
162 val = paravirt_read_msr(msr, &_err); \
165 #define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
166 #define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b)
168 /* rdmsr with exception handling */
169 #define rdmsr_safe(msr, a, b) \
172 u64 _l = paravirt_read_msr(msr, &_err); \
178 #define rdmsr_safe_regs(regs) paravirt_rdmsr_regs(regs)
179 #define wrmsr_safe_regs(regs) paravirt_wrmsr_regs(regs)
181 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
185 *p = paravirt_read_msr(msr, &err);
188 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
194 gprs[7] = 0x9c5a203a;
196 err = paravirt_rdmsr_regs(gprs);
198 *p = gprs[0] | ((u64)gprs[2] << 32);
203 static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
210 gprs[7] = 0x9c5a203a;
212 return paravirt_wrmsr_regs(gprs);
215 static inline u64 paravirt_read_tsc(void)
217 return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
220 #define rdtscl(low) \
222 u64 _l = paravirt_read_tsc(); \
226 #define rdtscll(val) (val = paravirt_read_tsc())
228 static inline unsigned long long paravirt_sched_clock(void)
230 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
233 static inline unsigned long long paravirt_read_pmc(int counter)
235 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
238 #define rdpmc(counter, low, high) \
240 u64 _l = paravirt_read_pmc(counter); \
245 static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
247 return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
250 #define rdtscp(low, high, aux) \
253 unsigned long __val = paravirt_rdtscp(&__aux); \
254 (low) = (u32)__val; \
255 (high) = (u32)(__val >> 32); \
259 #define rdtscpll(val, aux) \
261 unsigned long __aux; \
262 val = paravirt_rdtscp(&__aux); \
266 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
268 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
271 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
273 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
276 static inline void load_TR_desc(void)
278 PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
280 static inline void load_gdt(const struct desc_ptr *dtr)
282 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
284 static inline void load_idt(const struct desc_ptr *dtr)
286 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
288 static inline void set_ldt(const void *addr, unsigned entries)
290 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
292 static inline void store_gdt(struct desc_ptr *dtr)
294 PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
296 static inline void store_idt(struct desc_ptr *dtr)
298 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
300 static inline unsigned long paravirt_store_tr(void)
302 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
304 #define store_tr(tr) ((tr) = paravirt_store_tr())
305 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
307 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
311 static inline void load_gs_index(unsigned int gs)
313 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
317 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
320 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
323 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
324 void *desc, int type)
326 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
329 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
331 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
333 static inline void set_iopl_mask(unsigned mask)
335 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
338 /* The paravirtualized I/O functions */
339 static inline void slow_down_io(void)
341 pv_cpu_ops.io_delay();
342 #ifdef REALLY_SLOW_IO
343 pv_cpu_ops.io_delay();
344 pv_cpu_ops.io_delay();
345 pv_cpu_ops.io_delay();
350 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
351 unsigned long start_esp)
353 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
354 phys_apicid, start_eip, start_esp);
358 static inline void paravirt_activate_mm(struct mm_struct *prev,
359 struct mm_struct *next)
361 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
364 static inline void arch_dup_mmap(struct mm_struct *oldmm,
365 struct mm_struct *mm)
367 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
370 static inline void arch_exit_mmap(struct mm_struct *mm)
372 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
375 static inline void __flush_tlb(void)
377 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
379 static inline void __flush_tlb_global(void)
381 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
383 static inline void __flush_tlb_single(unsigned long addr)
385 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
388 static inline void flush_tlb_others(const struct cpumask *cpumask,
389 struct mm_struct *mm,
392 PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
395 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
397 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
400 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
402 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
405 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
407 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
409 static inline void paravirt_release_pte(unsigned long pfn)
411 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
414 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
416 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
419 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
420 unsigned long start, unsigned long count)
422 PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
424 static inline void paravirt_release_pmd(unsigned long pfn)
426 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
429 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
431 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
433 static inline void paravirt_release_pud(unsigned long pfn)
435 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
438 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
441 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
444 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
447 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
450 static inline pte_t __pte(pteval_t val)
454 if (sizeof(pteval_t) > sizeof(long))
455 ret = PVOP_CALLEE2(pteval_t,
457 val, (u64)val >> 32);
459 ret = PVOP_CALLEE1(pteval_t,
463 return (pte_t) { .pte = ret };
466 static inline pteval_t pte_val(pte_t pte)
470 if (sizeof(pteval_t) > sizeof(long))
471 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
472 pte.pte, (u64)pte.pte >> 32);
474 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
480 static inline pgd_t __pgd(pgdval_t val)
484 if (sizeof(pgdval_t) > sizeof(long))
485 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
486 val, (u64)val >> 32);
488 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
491 return (pgd_t) { ret };
494 static inline pgdval_t pgd_val(pgd_t pgd)
498 if (sizeof(pgdval_t) > sizeof(long))
499 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
500 pgd.pgd, (u64)pgd.pgd >> 32);
502 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
508 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
509 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
514 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
517 return (pte_t) { .pte = ret };
520 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
521 pte_t *ptep, pte_t pte)
523 if (sizeof(pteval_t) > sizeof(long))
525 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
527 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
528 mm, addr, ptep, pte.pte);
531 static inline void set_pte(pte_t *ptep, pte_t pte)
533 if (sizeof(pteval_t) > sizeof(long))
534 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
535 pte.pte, (u64)pte.pte >> 32);
537 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
541 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
542 pte_t *ptep, pte_t pte)
544 if (sizeof(pteval_t) > sizeof(long))
546 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
548 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
551 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
553 pmdval_t val = native_pmd_val(pmd);
555 if (sizeof(pmdval_t) > sizeof(long))
556 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
558 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
561 #if PAGETABLE_LEVELS >= 3
562 static inline pmd_t __pmd(pmdval_t val)
566 if (sizeof(pmdval_t) > sizeof(long))
567 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
568 val, (u64)val >> 32);
570 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
573 return (pmd_t) { ret };
576 static inline pmdval_t pmd_val(pmd_t pmd)
580 if (sizeof(pmdval_t) > sizeof(long))
581 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
582 pmd.pmd, (u64)pmd.pmd >> 32);
584 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
590 static inline void set_pud(pud_t *pudp, pud_t pud)
592 pudval_t val = native_pud_val(pud);
594 if (sizeof(pudval_t) > sizeof(long))
595 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
596 val, (u64)val >> 32);
598 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
601 #if PAGETABLE_LEVELS == 4
602 static inline pud_t __pud(pudval_t val)
606 if (sizeof(pudval_t) > sizeof(long))
607 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
608 val, (u64)val >> 32);
610 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
613 return (pud_t) { ret };
616 static inline pudval_t pud_val(pud_t pud)
620 if (sizeof(pudval_t) > sizeof(long))
621 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
622 pud.pud, (u64)pud.pud >> 32);
624 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
630 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
632 pgdval_t val = native_pgd_val(pgd);
634 if (sizeof(pgdval_t) > sizeof(long))
635 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
636 val, (u64)val >> 32);
638 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
642 static inline void pgd_clear(pgd_t *pgdp)
644 set_pgd(pgdp, __pgd(0));
647 static inline void pud_clear(pud_t *pudp)
649 set_pud(pudp, __pud(0));
652 #endif /* PAGETABLE_LEVELS == 4 */
654 #endif /* PAGETABLE_LEVELS >= 3 */
656 #ifdef CONFIG_X86_PAE
657 /* Special-case pte-setting operations for PAE, which can't update a
658 64-bit pte atomically */
659 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
661 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
662 pte.pte, pte.pte >> 32);
665 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
668 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
671 static inline void pmd_clear(pmd_t *pmdp)
673 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
675 #else /* !CONFIG_X86_PAE */
676 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
681 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
684 set_pte_at(mm, addr, ptep, __pte(0));
687 static inline void pmd_clear(pmd_t *pmdp)
689 set_pmd(pmdp, __pmd(0));
691 #endif /* CONFIG_X86_PAE */
693 #define __HAVE_ARCH_START_CONTEXT_SWITCH
694 static inline void arch_start_context_switch(struct task_struct *prev)
696 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
699 static inline void arch_end_context_switch(struct task_struct *next)
701 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
704 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
705 static inline void arch_enter_lazy_mmu_mode(void)
707 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
710 static inline void arch_leave_lazy_mmu_mode(void)
712 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
715 void arch_flush_lazy_mmu_mode(void);
717 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
718 phys_addr_t phys, pgprot_t flags)
720 pv_mmu_ops.set_fixmap(idx, phys, flags);
723 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
725 static inline int arch_spin_is_locked(struct arch_spinlock *lock)
727 return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);
730 static inline int arch_spin_is_contended(struct arch_spinlock *lock)
732 return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);
734 #define arch_spin_is_contended arch_spin_is_contended
736 static __always_inline void arch_spin_lock(struct arch_spinlock *lock)
738 PVOP_VCALL1(pv_lock_ops.spin_lock, lock);
741 static __always_inline void arch_spin_lock_flags(struct arch_spinlock *lock,
744 PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags);
747 static __always_inline int arch_spin_trylock(struct arch_spinlock *lock)
749 return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);
752 static __always_inline void arch_spin_unlock(struct arch_spinlock *lock)
754 PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);
760 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
761 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
763 /* save and restore all caller-save registers, except return value */
764 #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
765 #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"
767 #define PV_FLAGS_ARG "0"
768 #define PV_EXTRA_CLOBBERS
769 #define PV_VEXTRA_CLOBBERS
771 /* save and restore all caller-save registers, except return value */
772 #define PV_SAVE_ALL_CALLER_REGS \
781 #define PV_RESTORE_ALL_CALLER_REGS \
791 /* We save some registers, but all of them, that's too much. We clobber all
792 * caller saved registers but the argument parameter */
793 #define PV_SAVE_REGS "pushq %%rdi;"
794 #define PV_RESTORE_REGS "popq %%rdi;"
795 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
796 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
797 #define PV_FLAGS_ARG "D"
801 * Generate a thunk around a function which saves all caller-save
802 * registers except for the return value. This allows C functions to
803 * be called from assembler code where fewer than normal registers are
804 * available. It may also help code generation around calls from C
805 * code if the common case doesn't use many registers.
807 * When a callee is wrapped in a thunk, the caller can assume that all
808 * arg regs and all scratch registers are preserved across the
809 * call. The return value in rax/eax will not be saved, even for void
812 #define PV_CALLEE_SAVE_REGS_THUNK(func) \
813 extern typeof(func) __raw_callee_save_##func; \
814 static void *__##func##__ __used = func; \
816 asm(".pushsection .text;" \
817 "__raw_callee_save_" #func ": " \
818 PV_SAVE_ALL_CALLER_REGS \
820 PV_RESTORE_ALL_CALLER_REGS \
824 /* Get a reference to a callee-save function */
825 #define PV_CALLEE_SAVE(func) \
826 ((struct paravirt_callee_save) { __raw_callee_save_##func })
828 /* Promise that "func" already uses the right calling convention */
829 #define __PV_IS_CALLEE_SAVE(func) \
830 ((struct paravirt_callee_save) { func })
832 static inline unsigned long __raw_local_save_flags(void)
834 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl);
837 static inline void raw_local_irq_restore(unsigned long f)
839 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f);
842 static inline void raw_local_irq_disable(void)
844 PVOP_VCALLEE0(pv_irq_ops.irq_disable);
847 static inline void raw_local_irq_enable(void)
849 PVOP_VCALLEE0(pv_irq_ops.irq_enable);
852 static inline unsigned long __raw_local_irq_save(void)
856 f = __raw_local_save_flags();
857 raw_local_irq_disable();
862 /* Make sure as little as possible of this mess escapes. */
877 extern void default_banner(void);
879 #else /* __ASSEMBLY__ */
881 #define _PVSITE(ptype, clobbers, ops, word, algn) \
885 .pushsection .parainstructions,"a"; \
894 #define COND_PUSH(set, mask, reg) \
895 .if ((~(set)) & mask); push %reg; .endif
896 #define COND_POP(set, mask, reg) \
897 .if ((~(set)) & mask); pop %reg; .endif
901 #define PV_SAVE_REGS(set) \
902 COND_PUSH(set, CLBR_RAX, rax); \
903 COND_PUSH(set, CLBR_RCX, rcx); \
904 COND_PUSH(set, CLBR_RDX, rdx); \
905 COND_PUSH(set, CLBR_RSI, rsi); \
906 COND_PUSH(set, CLBR_RDI, rdi); \
907 COND_PUSH(set, CLBR_R8, r8); \
908 COND_PUSH(set, CLBR_R9, r9); \
909 COND_PUSH(set, CLBR_R10, r10); \
910 COND_PUSH(set, CLBR_R11, r11)
911 #define PV_RESTORE_REGS(set) \
912 COND_POP(set, CLBR_R11, r11); \
913 COND_POP(set, CLBR_R10, r10); \
914 COND_POP(set, CLBR_R9, r9); \
915 COND_POP(set, CLBR_R8, r8); \
916 COND_POP(set, CLBR_RDI, rdi); \
917 COND_POP(set, CLBR_RSI, rsi); \
918 COND_POP(set, CLBR_RDX, rdx); \
919 COND_POP(set, CLBR_RCX, rcx); \
920 COND_POP(set, CLBR_RAX, rax)
922 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
923 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
924 #define PARA_INDIRECT(addr) *addr(%rip)
926 #define PV_SAVE_REGS(set) \
927 COND_PUSH(set, CLBR_EAX, eax); \
928 COND_PUSH(set, CLBR_EDI, edi); \
929 COND_PUSH(set, CLBR_ECX, ecx); \
930 COND_PUSH(set, CLBR_EDX, edx)
931 #define PV_RESTORE_REGS(set) \
932 COND_POP(set, CLBR_EDX, edx); \
933 COND_POP(set, CLBR_ECX, ecx); \
934 COND_POP(set, CLBR_EDI, edi); \
935 COND_POP(set, CLBR_EAX, eax)
937 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
938 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
939 #define PARA_INDIRECT(addr) *%cs:addr
942 #define INTERRUPT_RETURN \
943 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
944 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
946 #define DISABLE_INTERRUPTS(clobbers) \
947 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
948 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
949 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
950 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
952 #define ENABLE_INTERRUPTS(clobbers) \
953 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
954 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
955 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
956 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
958 #define USERGS_SYSRET32 \
959 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
961 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
964 #define GET_CR0_INTO_EAX \
965 push %ecx; push %edx; \
966 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
969 #define ENABLE_INTERRUPTS_SYSEXIT \
970 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
972 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
975 #else /* !CONFIG_X86_32 */
978 * If swapgs is used while the userspace stack is still current,
979 * there's no way to call a pvop. The PV replacement *must* be
980 * inlined, or the swapgs instruction must be trapped and emulated.
982 #define SWAPGS_UNSAFE_STACK \
983 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
987 * Note: swapgs is very special, and in practise is either going to be
988 * implemented with a single "swapgs" instruction or something very
989 * special. Either way, we don't need to save any registers for
993 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
994 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \
997 #define GET_CR2_INTO_RCX \
998 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
1002 #define PARAVIRT_ADJUST_EXCEPTION_FRAME \
1003 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
1005 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
1007 #define USERGS_SYSRET64 \
1008 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \
1010 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
1012 #define ENABLE_INTERRUPTS_SYSEXIT32 \
1013 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
1015 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1016 #endif /* CONFIG_X86_32 */
1018 #endif /* __ASSEMBLY__ */
1019 #else /* CONFIG_PARAVIRT */
1020 # define default_banner x86_init_noop
1021 #endif /* !CONFIG_PARAVIRT */
1022 #endif /* _ASM_X86_PARAVIRT_H */