1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include "kvm_emulate.h"
24 #include <linux/stringify.h>
25 #include <asm/debugreg.h>
26 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
179 #define IsBranch ((u64)1 << 56) /* Instruction is considered a branch. */
181 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
183 #define X2(x...) x, x
184 #define X3(x...) X2(x), x
185 #define X4(x...) X2(x), X2(x)
186 #define X5(x...) X4(x), x
187 #define X6(x...) X4(x), X2(x)
188 #define X7(x...) X4(x), X3(x)
189 #define X8(x...) X4(x), X4(x)
190 #define X16(x...) X8(x), X8(x)
197 int (*execute)(struct x86_emulate_ctxt *ctxt);
198 const struct opcode *group;
199 const struct group_dual *gdual;
200 const struct gprefix *gprefix;
201 const struct escape *esc;
202 const struct instr_dual *idual;
203 const struct mode_dual *mdual;
204 void (*fastop)(struct fastop *fake);
206 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
210 struct opcode mod012[8];
211 struct opcode mod3[8];
215 struct opcode pfx_no;
216 struct opcode pfx_66;
217 struct opcode pfx_f2;
218 struct opcode pfx_f3;
223 struct opcode high[64];
227 struct opcode mod012;
232 struct opcode mode32;
233 struct opcode mode64;
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
238 enum x86_transfer_type {
240 X86_TRANSFER_CALL_JMP,
242 X86_TRANSFER_TASK_SWITCH,
245 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
247 if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))
248 nr &= NR_EMULATOR_GPRS - 1;
250 if (!(ctxt->regs_valid & (1 << nr))) {
251 ctxt->regs_valid |= 1 << nr;
252 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
254 return ctxt->_regs[nr];
257 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
259 if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))
260 nr &= NR_EMULATOR_GPRS - 1;
262 BUILD_BUG_ON(sizeof(ctxt->regs_dirty) * BITS_PER_BYTE < NR_EMULATOR_GPRS);
263 BUILD_BUG_ON(sizeof(ctxt->regs_valid) * BITS_PER_BYTE < NR_EMULATOR_GPRS);
265 ctxt->regs_valid |= 1 << nr;
266 ctxt->regs_dirty |= 1 << nr;
267 return &ctxt->_regs[nr];
270 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
273 return reg_write(ctxt, nr);
276 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
278 unsigned long dirty = ctxt->regs_dirty;
281 for_each_set_bit(reg, &dirty, NR_EMULATOR_GPRS)
282 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
285 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
287 ctxt->regs_dirty = 0;
288 ctxt->regs_valid = 0;
292 * These EFLAGS bits are restored from saved value during emulation, and
293 * any changes are written back to the saved value after emulation.
295 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
296 X86_EFLAGS_PF|X86_EFLAGS_CF)
305 * fastop functions have a special calling convention:
310 * flags: rflags (in/out)
311 * ex: rsi (in:fastop pointer, out:zero if exception)
313 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
314 * different operand sizes can be reached by calculation, rather than a jump
315 * table (which would be bigger than the code).
317 * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR
318 * and 1 for the straight line speculation INT3, leaves 7 bytes for the
319 * body of the function. Currently none is larger than 4.
321 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
323 #define FASTOP_SIZE 16
325 #define __FOP_FUNC(name) \
326 ".align " __stringify(FASTOP_SIZE) " \n\t" \
327 ".type " name ", @function \n\t" \
331 #define FOP_FUNC(name) \
334 #define __FOP_RET(name) \
336 ".size " name ", .-" name "\n\t"
338 #define FOP_RET(name) \
341 #define __FOP_START(op, align) \
342 extern void em_##op(struct fastop *fake); \
343 asm(".pushsection .text, \"ax\" \n\t" \
344 ".global em_" #op " \n\t" \
345 ".align " __stringify(align) " \n\t" \
348 #define FOP_START(op) __FOP_START(op, FASTOP_SIZE)
353 #define __FOPNOP(name) \
358 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
360 #define FOP1E(op, dst) \
361 __FOP_FUNC(#op "_" #dst) \
362 "10: " #op " %" #dst " \n\t" \
363 __FOP_RET(#op "_" #dst)
365 #define FOP1EEX(op, dst) \
366 FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
368 #define FASTOP1(op) \
373 ON64(FOP1E(op##q, rax)) \
376 /* 1-operand, using src2 (for MUL/DIV r/m) */
377 #define FASTOP1SRC2(op, name) \
382 ON64(FOP1E(op, rcx)) \
385 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
386 #define FASTOP1SRC2EX(op, name) \
391 ON64(FOP1EEX(op, rcx)) \
394 #define FOP2E(op, dst, src) \
395 __FOP_FUNC(#op "_" #dst "_" #src) \
396 #op " %" #src ", %" #dst " \n\t" \
397 __FOP_RET(#op "_" #dst "_" #src)
399 #define FASTOP2(op) \
401 FOP2E(op##b, al, dl) \
402 FOP2E(op##w, ax, dx) \
403 FOP2E(op##l, eax, edx) \
404 ON64(FOP2E(op##q, rax, rdx)) \
407 /* 2 operand, word only */
408 #define FASTOP2W(op) \
411 FOP2E(op##w, ax, dx) \
412 FOP2E(op##l, eax, edx) \
413 ON64(FOP2E(op##q, rax, rdx)) \
416 /* 2 operand, src is CL */
417 #define FASTOP2CL(op) \
419 FOP2E(op##b, al, cl) \
420 FOP2E(op##w, ax, cl) \
421 FOP2E(op##l, eax, cl) \
422 ON64(FOP2E(op##q, rax, cl)) \
425 /* 2 operand, src and dest are reversed */
426 #define FASTOP2R(op, name) \
428 FOP2E(op##b, dl, al) \
429 FOP2E(op##w, dx, ax) \
430 FOP2E(op##l, edx, eax) \
431 ON64(FOP2E(op##q, rdx, rax)) \
434 #define FOP3E(op, dst, src, src2) \
435 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
436 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
437 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
439 /* 3-operand, word-only, src2=cl */
440 #define FASTOP3WCL(op) \
443 FOP3E(op##w, ax, dx, cl) \
444 FOP3E(op##l, eax, edx, cl) \
445 ON64(FOP3E(op##q, rax, rdx, cl)) \
448 /* Special case for SETcc - 1 instruction per cc */
451 * Depending on .config the SETcc functions look like:
453 * ENDBR [4 bytes; CONFIG_X86_KERNEL_IBT]
454 * SETcc %al [3 bytes]
455 * RET | JMP __x86_return_thunk [1,5 bytes; CONFIG_RETHUNK]
456 * INT3 [1 byte; CONFIG_SLS]
458 #define SETCC_ALIGN 16
460 #define FOP_SETCC(op) \
461 ".align " __stringify(SETCC_ALIGN) " \n\t" \
462 ".type " #op ", @function \n\t" \
467 ".skip " __stringify(SETCC_ALIGN) " - (.-" #op "), 0xcc \n\t"
469 __FOP_START(setcc, SETCC_ALIGN)
490 "pushf; sbb %al, %al; popf \n\t"
495 * XXX: inoutclob user must know where the argument is being expanded.
496 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
498 #define asm_safe(insn, inoutclob...) \
502 asm volatile("1:" insn "\n" \
504 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[_fault]) \
505 : [_fault] "+r"(_fault) inoutclob ); \
507 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
510 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
511 enum x86_intercept intercept,
512 enum x86_intercept_stage stage)
514 struct x86_instruction_info info = {
515 .intercept = intercept,
516 .rep_prefix = ctxt->rep_prefix,
517 .modrm_mod = ctxt->modrm_mod,
518 .modrm_reg = ctxt->modrm_reg,
519 .modrm_rm = ctxt->modrm_rm,
520 .src_val = ctxt->src.val64,
521 .dst_val = ctxt->dst.val64,
522 .src_bytes = ctxt->src.bytes,
523 .dst_bytes = ctxt->dst.bytes,
524 .ad_bytes = ctxt->ad_bytes,
525 .next_rip = ctxt->eip,
528 return ctxt->ops->intercept(ctxt, &info, stage);
531 static void assign_masked(ulong *dest, ulong src, ulong mask)
533 *dest = (*dest & ~mask) | (src & mask);
536 static void assign_register(unsigned long *reg, u64 val, int bytes)
538 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
541 *(u8 *)reg = (u8)val;
544 *(u16 *)reg = (u16)val;
548 break; /* 64b: zero-extend */
555 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
557 return (1UL << (ctxt->ad_bytes << 3)) - 1;
560 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
563 struct desc_struct ss;
565 if (ctxt->mode == X86EMUL_MODE_PROT64)
567 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
568 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
571 static int stack_size(struct x86_emulate_ctxt *ctxt)
573 return (__fls(stack_mask(ctxt)) + 1) >> 3;
576 /* Access/update address held in a register, based on addressing mode. */
577 static inline unsigned long
578 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
580 if (ctxt->ad_bytes == sizeof(unsigned long))
583 return reg & ad_mask(ctxt);
586 static inline unsigned long
587 register_address(struct x86_emulate_ctxt *ctxt, int reg)
589 return address_mask(ctxt, reg_read(ctxt, reg));
592 static void masked_increment(ulong *reg, ulong mask, int inc)
594 assign_masked(reg, *reg + inc, mask);
598 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
600 ulong *preg = reg_rmw(ctxt, reg);
602 assign_register(preg, *preg + inc, ctxt->ad_bytes);
605 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
607 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
610 static u32 desc_limit_scaled(struct desc_struct *desc)
612 u32 limit = get_desc_limit(desc);
614 return desc->g ? (limit << 12) | 0xfff : limit;
617 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
619 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
622 return ctxt->ops->get_cached_segment_base(ctxt, seg);
625 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
626 u32 error, bool valid)
628 if (KVM_EMULATOR_BUG_ON(vec > 0x1f, ctxt))
629 return X86EMUL_UNHANDLEABLE;
631 ctxt->exception.vector = vec;
632 ctxt->exception.error_code = error;
633 ctxt->exception.error_code_valid = valid;
634 return X86EMUL_PROPAGATE_FAULT;
637 static int emulate_db(struct x86_emulate_ctxt *ctxt)
639 return emulate_exception(ctxt, DB_VECTOR, 0, false);
642 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
644 return emulate_exception(ctxt, GP_VECTOR, err, true);
647 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
649 return emulate_exception(ctxt, SS_VECTOR, err, true);
652 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
654 return emulate_exception(ctxt, UD_VECTOR, 0, false);
657 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
659 return emulate_exception(ctxt, TS_VECTOR, err, true);
662 static int emulate_de(struct x86_emulate_ctxt *ctxt)
664 return emulate_exception(ctxt, DE_VECTOR, 0, false);
667 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
669 return emulate_exception(ctxt, NM_VECTOR, 0, false);
672 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
675 struct desc_struct desc;
677 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
681 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
686 struct desc_struct desc;
688 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
689 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
692 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
694 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
697 static inline bool emul_is_noncanonical_address(u64 la,
698 struct x86_emulate_ctxt *ctxt)
700 return !__is_canonical_address(la, ctxt_virt_addr_bits(ctxt));
704 * x86 defines three classes of vector instructions: explicitly
705 * aligned, explicitly unaligned, and the rest, which change behaviour
706 * depending on whether they're AVX encoded or not.
708 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
709 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
710 * 512 bytes of data must be aligned to a 16 byte boundary.
712 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
714 u64 alignment = ctxt->d & AlignMask;
716 if (likely(size < 16))
731 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
732 struct segmented_address addr,
733 unsigned *max_size, unsigned size,
734 bool write, bool fetch,
735 enum x86emul_mode mode, ulong *linear)
737 struct desc_struct desc;
744 la = seg_base(ctxt, addr.seg) + addr.ea;
747 case X86EMUL_MODE_PROT64:
749 va_bits = ctxt_virt_addr_bits(ctxt);
750 if (!__is_canonical_address(la, va_bits))
753 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
754 if (size > *max_size)
758 *linear = la = (u32)la;
759 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
763 /* code segment in protected mode or read-only data segment */
764 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
765 || !(desc.type & 2)) && write)
767 /* unreadable code segment */
768 if (!fetch && (desc.type & 8) && !(desc.type & 2))
770 lim = desc_limit_scaled(&desc);
771 if (!(desc.type & 8) && (desc.type & 4)) {
772 /* expand-down segment */
775 lim = desc.d ? 0xffffffff : 0xffff;
779 if (lim == 0xffffffff)
782 *max_size = (u64)lim + 1 - addr.ea;
783 if (size > *max_size)
788 if (la & (insn_alignment(ctxt, size) - 1))
789 return emulate_gp(ctxt, 0);
790 return X86EMUL_CONTINUE;
792 if (addr.seg == VCPU_SREG_SS)
793 return emulate_ss(ctxt, 0);
795 return emulate_gp(ctxt, 0);
798 static int linearize(struct x86_emulate_ctxt *ctxt,
799 struct segmented_address addr,
800 unsigned size, bool write,
804 return __linearize(ctxt, addr, &max_size, size, write, false,
808 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
809 enum x86emul_mode mode)
814 struct segmented_address addr = { .seg = VCPU_SREG_CS,
817 if (ctxt->op_bytes != sizeof(unsigned long))
818 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
819 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
820 if (rc == X86EMUL_CONTINUE)
821 ctxt->_eip = addr.ea;
825 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
827 return assign_eip(ctxt, dst, ctxt->mode);
830 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
831 const struct desc_struct *cs_desc)
833 enum x86emul_mode mode = ctxt->mode;
837 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
841 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
843 mode = X86EMUL_MODE_PROT64;
845 mode = X86EMUL_MODE_PROT32; /* temporary value */
848 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
849 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
850 rc = assign_eip(ctxt, dst, mode);
851 if (rc == X86EMUL_CONTINUE)
856 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
858 return assign_eip_near(ctxt, ctxt->_eip + rel);
861 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
862 void *data, unsigned size)
864 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
867 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
868 ulong linear, void *data,
871 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
874 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
875 struct segmented_address addr,
882 rc = linearize(ctxt, addr, size, false, &linear);
883 if (rc != X86EMUL_CONTINUE)
885 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
888 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
889 struct segmented_address addr,
896 rc = linearize(ctxt, addr, size, true, &linear);
897 if (rc != X86EMUL_CONTINUE)
899 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
903 * Prefetch the remaining bytes of the instruction without crossing page
904 * boundary if they are not in fetch_cache yet.
906 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
909 unsigned size, max_size;
910 unsigned long linear;
911 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
912 struct segmented_address addr = { .seg = VCPU_SREG_CS,
913 .ea = ctxt->eip + cur_size };
916 * We do not know exactly how many bytes will be needed, and
917 * __linearize is expensive, so fetch as much as possible. We
918 * just have to avoid going beyond the 15 byte limit, the end
919 * of the segment, or the end of the page.
921 * __linearize is called with size 0 so that it does not do any
922 * boundary check itself. Instead, we use max_size to check
925 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
927 if (unlikely(rc != X86EMUL_CONTINUE))
930 size = min_t(unsigned, 15UL ^ cur_size, max_size);
931 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
934 * One instruction can only straddle two pages,
935 * and one has been loaded at the beginning of
936 * x86_decode_insn. So, if not enough bytes
937 * still, we must have hit the 15-byte boundary.
939 if (unlikely(size < op_size))
940 return emulate_gp(ctxt, 0);
942 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
943 size, &ctxt->exception);
944 if (unlikely(rc != X86EMUL_CONTINUE))
946 ctxt->fetch.end += size;
947 return X86EMUL_CONTINUE;
950 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
953 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
955 if (unlikely(done_size < size))
956 return __do_insn_fetch_bytes(ctxt, size - done_size);
958 return X86EMUL_CONTINUE;
961 /* Fetch next part of the instruction being emulated. */
962 #define insn_fetch(_type, _ctxt) \
965 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
966 if (rc != X86EMUL_CONTINUE) \
968 ctxt->_eip += sizeof(_type); \
969 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
970 ctxt->fetch.ptr += sizeof(_type); \
974 #define insn_fetch_arr(_arr, _size, _ctxt) \
976 rc = do_insn_fetch_bytes(_ctxt, _size); \
977 if (rc != X86EMUL_CONTINUE) \
979 ctxt->_eip += (_size); \
980 memcpy(_arr, ctxt->fetch.ptr, _size); \
981 ctxt->fetch.ptr += (_size); \
985 * Given the 'reg' portion of a ModRM byte, and a register block, return a
986 * pointer into the block that addresses the relevant register.
987 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
989 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
993 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
995 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
996 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
998 p = reg_rmw(ctxt, modrm_reg);
1002 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
1003 struct segmented_address addr,
1004 u16 *size, unsigned long *address, int op_bytes)
1011 rc = segmented_read_std(ctxt, addr, size, 2);
1012 if (rc != X86EMUL_CONTINUE)
1015 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1029 FASTOP1SRC2(mul, mul_ex);
1030 FASTOP1SRC2(imul, imul_ex);
1031 FASTOP1SRC2EX(div, div_ex);
1032 FASTOP1SRC2EX(idiv, idiv_ex);
1061 FASTOP2R(cmp, cmp_r);
1063 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1065 /* If src is zero, do not writeback, but update flags */
1066 if (ctxt->src.val == 0)
1067 ctxt->dst.type = OP_NONE;
1068 return fastop(ctxt, em_bsf);
1071 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1073 /* If src is zero, do not writeback, but update flags */
1074 if (ctxt->src.val == 0)
1075 ctxt->dst.type = OP_NONE;
1076 return fastop(ctxt, em_bsr);
1079 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1082 void (*fop)(void) = (void *)em_setcc + SETCC_ALIGN * (condition & 0xf);
1084 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1085 asm("push %[flags]; popf; " CALL_NOSPEC
1086 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1090 static void fetch_register_operand(struct operand *op)
1092 switch (op->bytes) {
1094 op->val = *(u8 *)op->addr.reg;
1097 op->val = *(u16 *)op->addr.reg;
1100 op->val = *(u32 *)op->addr.reg;
1103 op->val = *(u64 *)op->addr.reg;
1108 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1110 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1111 return emulate_nm(ctxt);
1114 asm volatile("fninit");
1116 return X86EMUL_CONTINUE;
1119 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1123 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1124 return emulate_nm(ctxt);
1127 asm volatile("fnstcw %0": "+m"(fcw));
1130 ctxt->dst.val = fcw;
1132 return X86EMUL_CONTINUE;
1135 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1139 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1140 return emulate_nm(ctxt);
1143 asm volatile("fnstsw %0": "+m"(fsw));
1146 ctxt->dst.val = fsw;
1148 return X86EMUL_CONTINUE;
1151 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1154 unsigned reg = ctxt->modrm_reg;
1156 if (!(ctxt->d & ModRM))
1157 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1159 if (ctxt->d & Sse) {
1163 kvm_read_sse_reg(reg, &op->vec_val);
1166 if (ctxt->d & Mmx) {
1175 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1176 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1178 fetch_register_operand(op);
1179 op->orig_val = op->val;
1182 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1184 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1185 ctxt->modrm_seg = VCPU_SREG_SS;
1188 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1192 int index_reg, base_reg, scale;
1193 int rc = X86EMUL_CONTINUE;
1196 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1197 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1198 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1200 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1201 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1202 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1203 ctxt->modrm_seg = VCPU_SREG_DS;
1205 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1207 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1208 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1210 if (ctxt->d & Sse) {
1213 op->addr.xmm = ctxt->modrm_rm;
1214 kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);
1217 if (ctxt->d & Mmx) {
1220 op->addr.mm = ctxt->modrm_rm & 7;
1223 fetch_register_operand(op);
1229 if (ctxt->ad_bytes == 2) {
1230 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1231 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1232 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1233 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1235 /* 16-bit ModR/M decode. */
1236 switch (ctxt->modrm_mod) {
1238 if (ctxt->modrm_rm == 6)
1239 modrm_ea += insn_fetch(u16, ctxt);
1242 modrm_ea += insn_fetch(s8, ctxt);
1245 modrm_ea += insn_fetch(u16, ctxt);
1248 switch (ctxt->modrm_rm) {
1250 modrm_ea += bx + si;
1253 modrm_ea += bx + di;
1256 modrm_ea += bp + si;
1259 modrm_ea += bp + di;
1268 if (ctxt->modrm_mod != 0)
1275 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1276 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1277 ctxt->modrm_seg = VCPU_SREG_SS;
1278 modrm_ea = (u16)modrm_ea;
1280 /* 32/64-bit ModR/M decode. */
1281 if ((ctxt->modrm_rm & 7) == 4) {
1282 sib = insn_fetch(u8, ctxt);
1283 index_reg |= (sib >> 3) & 7;
1284 base_reg |= sib & 7;
1287 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1288 modrm_ea += insn_fetch(s32, ctxt);
1290 modrm_ea += reg_read(ctxt, base_reg);
1291 adjust_modrm_seg(ctxt, base_reg);
1292 /* Increment ESP on POP [ESP] */
1293 if ((ctxt->d & IncSP) &&
1294 base_reg == VCPU_REGS_RSP)
1295 modrm_ea += ctxt->op_bytes;
1298 modrm_ea += reg_read(ctxt, index_reg) << scale;
1299 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1300 modrm_ea += insn_fetch(s32, ctxt);
1301 if (ctxt->mode == X86EMUL_MODE_PROT64)
1302 ctxt->rip_relative = 1;
1304 base_reg = ctxt->modrm_rm;
1305 modrm_ea += reg_read(ctxt, base_reg);
1306 adjust_modrm_seg(ctxt, base_reg);
1308 switch (ctxt->modrm_mod) {
1310 modrm_ea += insn_fetch(s8, ctxt);
1313 modrm_ea += insn_fetch(s32, ctxt);
1317 op->addr.mem.ea = modrm_ea;
1318 if (ctxt->ad_bytes != 8)
1319 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1325 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1328 int rc = X86EMUL_CONTINUE;
1331 switch (ctxt->ad_bytes) {
1333 op->addr.mem.ea = insn_fetch(u16, ctxt);
1336 op->addr.mem.ea = insn_fetch(u32, ctxt);
1339 op->addr.mem.ea = insn_fetch(u64, ctxt);
1346 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1350 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1351 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1353 if (ctxt->src.bytes == 2)
1354 sv = (s16)ctxt->src.val & (s16)mask;
1355 else if (ctxt->src.bytes == 4)
1356 sv = (s32)ctxt->src.val & (s32)mask;
1358 sv = (s64)ctxt->src.val & (s64)mask;
1360 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1361 ctxt->dst.addr.mem.ea + (sv >> 3));
1364 /* only subword offset */
1365 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1368 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1369 unsigned long addr, void *dest, unsigned size)
1372 struct read_cache *mc = &ctxt->mem_read;
1374 if (mc->pos < mc->end)
1377 if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))
1378 return X86EMUL_UNHANDLEABLE;
1380 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1382 if (rc != X86EMUL_CONTINUE)
1388 memcpy(dest, mc->data + mc->pos, size);
1390 return X86EMUL_CONTINUE;
1393 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1394 struct segmented_address addr,
1401 rc = linearize(ctxt, addr, size, false, &linear);
1402 if (rc != X86EMUL_CONTINUE)
1404 return read_emulated(ctxt, linear, data, size);
1407 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1408 struct segmented_address addr,
1415 rc = linearize(ctxt, addr, size, true, &linear);
1416 if (rc != X86EMUL_CONTINUE)
1418 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1422 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1423 struct segmented_address addr,
1424 const void *orig_data, const void *data,
1430 rc = linearize(ctxt, addr, size, true, &linear);
1431 if (rc != X86EMUL_CONTINUE)
1433 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1434 size, &ctxt->exception);
1437 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1438 unsigned int size, unsigned short port,
1441 struct read_cache *rc = &ctxt->io_read;
1443 if (rc->pos == rc->end) { /* refill pio read ahead */
1444 unsigned int in_page, n;
1445 unsigned int count = ctxt->rep_prefix ?
1446 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1447 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1448 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1449 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1450 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1453 rc->pos = rc->end = 0;
1454 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1459 if (ctxt->rep_prefix && (ctxt->d & String) &&
1460 !(ctxt->eflags & X86_EFLAGS_DF)) {
1461 ctxt->dst.data = rc->data + rc->pos;
1462 ctxt->dst.type = OP_MEM_STR;
1463 ctxt->dst.count = (rc->end - rc->pos) / size;
1466 memcpy(dest, rc->data + rc->pos, size);
1472 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1473 u16 index, struct desc_struct *desc)
1478 ctxt->ops->get_idt(ctxt, &dt);
1480 if (dt.size < index * 8 + 7)
1481 return emulate_gp(ctxt, index << 3 | 0x2);
1483 addr = dt.address + index * 8;
1484 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1487 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1488 u16 selector, struct desc_ptr *dt)
1490 const struct x86_emulate_ops *ops = ctxt->ops;
1493 if (selector & 1 << 2) {
1494 struct desc_struct desc;
1497 memset(dt, 0, sizeof(*dt));
1498 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1502 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1503 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1505 ops->get_gdt(ctxt, dt);
1508 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1509 u16 selector, ulong *desc_addr_p)
1512 u16 index = selector >> 3;
1515 get_descriptor_table_ptr(ctxt, selector, &dt);
1517 if (dt.size < index * 8 + 7)
1518 return emulate_gp(ctxt, selector & 0xfffc);
1520 addr = dt.address + index * 8;
1522 #ifdef CONFIG_X86_64
1523 if (addr >> 32 != 0) {
1526 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1527 if (!(efer & EFER_LMA))
1532 *desc_addr_p = addr;
1533 return X86EMUL_CONTINUE;
1536 /* allowed just for 8 bytes segments */
1537 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1538 u16 selector, struct desc_struct *desc,
1543 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1544 if (rc != X86EMUL_CONTINUE)
1547 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1550 /* allowed just for 8 bytes segments */
1551 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1552 u16 selector, struct desc_struct *desc)
1557 rc = get_descriptor_ptr(ctxt, selector, &addr);
1558 if (rc != X86EMUL_CONTINUE)
1561 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1564 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1565 u16 selector, int seg, u8 cpl,
1566 enum x86_transfer_type transfer,
1567 struct desc_struct *desc)
1569 struct desc_struct seg_desc, old_desc;
1571 unsigned err_vec = GP_VECTOR;
1573 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1579 memset(&seg_desc, 0, sizeof(seg_desc));
1581 if (ctxt->mode == X86EMUL_MODE_REAL) {
1582 /* set real mode segment descriptor (keep limit etc. for
1584 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1585 set_desc_base(&seg_desc, selector << 4);
1587 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1588 /* VM86 needs a clean new segment descriptor */
1589 set_desc_base(&seg_desc, selector << 4);
1590 set_desc_limit(&seg_desc, 0xffff);
1600 /* TR should be in GDT only */
1601 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1604 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1605 if (null_selector) {
1606 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1609 if (seg == VCPU_SREG_SS) {
1610 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1614 * ctxt->ops->set_segment expects the CPL to be in
1615 * SS.DPL, so fake an expand-up 32-bit data segment.
1625 /* Skip all following checks */
1629 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1630 if (ret != X86EMUL_CONTINUE)
1633 err_code = selector & 0xfffc;
1634 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1637 /* can't load system descriptor into segment selector */
1638 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1639 if (transfer == X86_TRANSFER_CALL_JMP)
1640 return X86EMUL_UNHANDLEABLE;
1649 * segment is not a writable data segment or segment
1650 * selector's RPL != CPL or segment selector's RPL != CPL
1652 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1656 if (!(seg_desc.type & 8))
1659 if (transfer == X86_TRANSFER_RET) {
1660 /* RET can never return to an inner privilege level. */
1663 /* Outer-privilege level return is not implemented */
1665 return X86EMUL_UNHANDLEABLE;
1667 if (transfer == X86_TRANSFER_RET || transfer == X86_TRANSFER_TASK_SWITCH) {
1668 if (seg_desc.type & 4) {
1677 } else { /* X86_TRANSFER_CALL_JMP */
1678 if (seg_desc.type & 4) {
1684 if (rpl > cpl || dpl != cpl)
1688 /* in long-mode d/b must be clear if l is set */
1689 if (seg_desc.d && seg_desc.l) {
1692 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1693 if (efer & EFER_LMA)
1697 /* CS(RPL) <- CPL */
1698 selector = (selector & 0xfffc) | cpl;
1701 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1704 case VCPU_SREG_LDTR:
1705 if (seg_desc.s || seg_desc.type != 2)
1708 default: /* DS, ES, FS, or GS */
1710 * segment is not a data or readable code segment or
1711 * ((segment is a data or nonconforming code segment)
1712 * and (both RPL and CPL > DPL))
1714 if ((seg_desc.type & 0xa) == 0x8 ||
1715 (((seg_desc.type & 0xc) != 0xc) &&
1716 (rpl > dpl && cpl > dpl)))
1722 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1727 /* mark segment as accessed */
1728 if (!(seg_desc.type & 1)) {
1730 ret = write_segment_descriptor(ctxt, selector,
1732 if (ret != X86EMUL_CONTINUE)
1735 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1736 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1737 if (ret != X86EMUL_CONTINUE)
1739 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1740 ((u64)base3 << 32), ctxt))
1741 return emulate_gp(ctxt, err_code);
1744 if (seg == VCPU_SREG_TR) {
1745 old_desc = seg_desc;
1746 seg_desc.type |= 2; /* busy */
1747 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1748 sizeof(seg_desc), &ctxt->exception);
1749 if (ret != X86EMUL_CONTINUE)
1753 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1756 return X86EMUL_CONTINUE;
1758 return emulate_exception(ctxt, err_vec, err_code, true);
1761 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1762 u16 selector, int seg)
1764 u8 cpl = ctxt->ops->cpl(ctxt);
1767 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1768 * they can load it at CPL<3 (Intel's manual says only LSS can,
1771 * However, the Intel manual says that putting IST=1/DPL=3 in
1772 * an interrupt gate will result in SS=3 (the AMD manual instead
1773 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1774 * and only forbid it here.
1776 if (seg == VCPU_SREG_SS && selector == 3 &&
1777 ctxt->mode == X86EMUL_MODE_PROT64)
1778 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1780 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1781 X86_TRANSFER_NONE, NULL);
1784 static void write_register_operand(struct operand *op)
1786 return assign_register(op->addr.reg, op->val, op->bytes);
1789 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1793 write_register_operand(op);
1796 if (ctxt->lock_prefix)
1797 return segmented_cmpxchg(ctxt,
1803 return segmented_write(ctxt,
1809 return segmented_write(ctxt,
1812 op->bytes * op->count);
1815 kvm_write_sse_reg(op->addr.xmm, &op->vec_val);
1818 kvm_write_mmx_reg(op->addr.mm, &op->mm_val);
1826 return X86EMUL_CONTINUE;
1829 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1831 struct segmented_address addr;
1833 rsp_increment(ctxt, -bytes);
1834 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1835 addr.seg = VCPU_SREG_SS;
1837 return segmented_write(ctxt, addr, data, bytes);
1840 static int em_push(struct x86_emulate_ctxt *ctxt)
1842 /* Disable writeback. */
1843 ctxt->dst.type = OP_NONE;
1844 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1847 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1848 void *dest, int len)
1851 struct segmented_address addr;
1853 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1854 addr.seg = VCPU_SREG_SS;
1855 rc = segmented_read(ctxt, addr, dest, len);
1856 if (rc != X86EMUL_CONTINUE)
1859 rsp_increment(ctxt, len);
1863 static int em_pop(struct x86_emulate_ctxt *ctxt)
1865 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1868 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1869 void *dest, int len)
1872 unsigned long val, change_mask;
1873 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1874 int cpl = ctxt->ops->cpl(ctxt);
1876 rc = emulate_pop(ctxt, &val, len);
1877 if (rc != X86EMUL_CONTINUE)
1880 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1881 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1882 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1883 X86_EFLAGS_AC | X86_EFLAGS_ID;
1885 switch(ctxt->mode) {
1886 case X86EMUL_MODE_PROT64:
1887 case X86EMUL_MODE_PROT32:
1888 case X86EMUL_MODE_PROT16:
1890 change_mask |= X86_EFLAGS_IOPL;
1892 change_mask |= X86_EFLAGS_IF;
1894 case X86EMUL_MODE_VM86:
1896 return emulate_gp(ctxt, 0);
1897 change_mask |= X86_EFLAGS_IF;
1899 default: /* real mode */
1900 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1904 *(unsigned long *)dest =
1905 (ctxt->eflags & ~change_mask) | (val & change_mask);
1910 static int em_popf(struct x86_emulate_ctxt *ctxt)
1912 ctxt->dst.type = OP_REG;
1913 ctxt->dst.addr.reg = &ctxt->eflags;
1914 ctxt->dst.bytes = ctxt->op_bytes;
1915 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1918 static int em_enter(struct x86_emulate_ctxt *ctxt)
1921 unsigned frame_size = ctxt->src.val;
1922 unsigned nesting_level = ctxt->src2.val & 31;
1926 return X86EMUL_UNHANDLEABLE;
1928 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1929 rc = push(ctxt, &rbp, stack_size(ctxt));
1930 if (rc != X86EMUL_CONTINUE)
1932 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1934 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1935 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1937 return X86EMUL_CONTINUE;
1940 static int em_leave(struct x86_emulate_ctxt *ctxt)
1942 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1944 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1947 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1949 int seg = ctxt->src2.val;
1951 ctxt->src.val = get_segment_selector(ctxt, seg);
1952 if (ctxt->op_bytes == 4) {
1953 rsp_increment(ctxt, -2);
1957 return em_push(ctxt);
1960 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1962 int seg = ctxt->src2.val;
1963 unsigned long selector;
1966 rc = emulate_pop(ctxt, &selector, 2);
1967 if (rc != X86EMUL_CONTINUE)
1970 if (ctxt->modrm_reg == VCPU_SREG_SS)
1971 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1972 if (ctxt->op_bytes > 2)
1973 rsp_increment(ctxt, ctxt->op_bytes - 2);
1975 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1979 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1981 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1982 int rc = X86EMUL_CONTINUE;
1983 int reg = VCPU_REGS_RAX;
1985 while (reg <= VCPU_REGS_RDI) {
1986 (reg == VCPU_REGS_RSP) ?
1987 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1990 if (rc != X86EMUL_CONTINUE)
1999 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2001 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2002 return em_push(ctxt);
2005 static int em_popa(struct x86_emulate_ctxt *ctxt)
2007 int rc = X86EMUL_CONTINUE;
2008 int reg = VCPU_REGS_RDI;
2011 while (reg >= VCPU_REGS_RAX) {
2012 if (reg == VCPU_REGS_RSP) {
2013 rsp_increment(ctxt, ctxt->op_bytes);
2017 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2018 if (rc != X86EMUL_CONTINUE)
2020 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2026 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2028 const struct x86_emulate_ops *ops = ctxt->ops;
2035 /* TODO: Add limit checks */
2036 ctxt->src.val = ctxt->eflags;
2038 if (rc != X86EMUL_CONTINUE)
2041 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2043 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2045 if (rc != X86EMUL_CONTINUE)
2048 ctxt->src.val = ctxt->_eip;
2050 if (rc != X86EMUL_CONTINUE)
2053 ops->get_idt(ctxt, &dt);
2055 eip_addr = dt.address + (irq << 2);
2056 cs_addr = dt.address + (irq << 2) + 2;
2058 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2059 if (rc != X86EMUL_CONTINUE)
2062 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2063 if (rc != X86EMUL_CONTINUE)
2066 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2067 if (rc != X86EMUL_CONTINUE)
2075 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2079 invalidate_registers(ctxt);
2080 rc = __emulate_int_real(ctxt, irq);
2081 if (rc == X86EMUL_CONTINUE)
2082 writeback_registers(ctxt);
2086 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2088 switch(ctxt->mode) {
2089 case X86EMUL_MODE_REAL:
2090 return __emulate_int_real(ctxt, irq);
2091 case X86EMUL_MODE_VM86:
2092 case X86EMUL_MODE_PROT16:
2093 case X86EMUL_MODE_PROT32:
2094 case X86EMUL_MODE_PROT64:
2096 /* Protected mode interrupts unimplemented yet */
2097 return X86EMUL_UNHANDLEABLE;
2101 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2103 int rc = X86EMUL_CONTINUE;
2104 unsigned long temp_eip = 0;
2105 unsigned long temp_eflags = 0;
2106 unsigned long cs = 0;
2107 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2108 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2109 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2110 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2111 X86_EFLAGS_AC | X86_EFLAGS_ID |
2113 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2116 /* TODO: Add stack limit check */
2118 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2120 if (rc != X86EMUL_CONTINUE)
2123 if (temp_eip & ~0xffff)
2124 return emulate_gp(ctxt, 0);
2126 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2128 if (rc != X86EMUL_CONTINUE)
2131 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2133 if (rc != X86EMUL_CONTINUE)
2136 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2138 if (rc != X86EMUL_CONTINUE)
2141 ctxt->_eip = temp_eip;
2143 if (ctxt->op_bytes == 4)
2144 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2145 else if (ctxt->op_bytes == 2) {
2146 ctxt->eflags &= ~0xffff;
2147 ctxt->eflags |= temp_eflags;
2150 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2151 ctxt->eflags |= X86_EFLAGS_FIXED;
2152 ctxt->ops->set_nmi_mask(ctxt, false);
2157 static int em_iret(struct x86_emulate_ctxt *ctxt)
2159 switch(ctxt->mode) {
2160 case X86EMUL_MODE_REAL:
2161 return emulate_iret_real(ctxt);
2162 case X86EMUL_MODE_VM86:
2163 case X86EMUL_MODE_PROT16:
2164 case X86EMUL_MODE_PROT32:
2165 case X86EMUL_MODE_PROT64:
2167 /* iret from protected mode unimplemented yet */
2168 return X86EMUL_UNHANDLEABLE;
2172 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2176 struct desc_struct new_desc;
2177 u8 cpl = ctxt->ops->cpl(ctxt);
2179 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2181 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2182 X86_TRANSFER_CALL_JMP,
2184 if (rc != X86EMUL_CONTINUE)
2187 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2188 /* Error handling is not implemented. */
2189 if (rc != X86EMUL_CONTINUE)
2190 return X86EMUL_UNHANDLEABLE;
2195 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2197 return assign_eip_near(ctxt, ctxt->src.val);
2200 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2205 old_eip = ctxt->_eip;
2206 rc = assign_eip_near(ctxt, ctxt->src.val);
2207 if (rc != X86EMUL_CONTINUE)
2209 ctxt->src.val = old_eip;
2214 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2216 u64 old = ctxt->dst.orig_val64;
2218 if (ctxt->dst.bytes == 16)
2219 return X86EMUL_UNHANDLEABLE;
2221 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2222 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2223 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2224 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2225 ctxt->eflags &= ~X86_EFLAGS_ZF;
2227 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2228 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2230 ctxt->eflags |= X86_EFLAGS_ZF;
2232 return X86EMUL_CONTINUE;
2235 static int em_ret(struct x86_emulate_ctxt *ctxt)
2240 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2241 if (rc != X86EMUL_CONTINUE)
2244 return assign_eip_near(ctxt, eip);
2247 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2250 unsigned long eip, cs;
2251 int cpl = ctxt->ops->cpl(ctxt);
2252 struct desc_struct new_desc;
2254 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2255 if (rc != X86EMUL_CONTINUE)
2257 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2258 if (rc != X86EMUL_CONTINUE)
2260 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2263 if (rc != X86EMUL_CONTINUE)
2265 rc = assign_eip_far(ctxt, eip, &new_desc);
2266 /* Error handling is not implemented. */
2267 if (rc != X86EMUL_CONTINUE)
2268 return X86EMUL_UNHANDLEABLE;
2273 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2277 rc = em_ret_far(ctxt);
2278 if (rc != X86EMUL_CONTINUE)
2280 rsp_increment(ctxt, ctxt->src.val);
2281 return X86EMUL_CONTINUE;
2284 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2286 /* Save real source value, then compare EAX against destination. */
2287 ctxt->dst.orig_val = ctxt->dst.val;
2288 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2289 ctxt->src.orig_val = ctxt->src.val;
2290 ctxt->src.val = ctxt->dst.orig_val;
2291 fastop(ctxt, em_cmp);
2293 if (ctxt->eflags & X86_EFLAGS_ZF) {
2294 /* Success: write back to memory; no update of EAX */
2295 ctxt->src.type = OP_NONE;
2296 ctxt->dst.val = ctxt->src.orig_val;
2298 /* Failure: write the value we saw to EAX. */
2299 ctxt->src.type = OP_REG;
2300 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2301 ctxt->src.val = ctxt->dst.orig_val;
2302 /* Create write-cycle to dest by writing the same value */
2303 ctxt->dst.val = ctxt->dst.orig_val;
2305 return X86EMUL_CONTINUE;
2308 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2310 int seg = ctxt->src2.val;
2314 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2316 rc = load_segment_descriptor(ctxt, sel, seg);
2317 if (rc != X86EMUL_CONTINUE)
2320 ctxt->dst.val = ctxt->src.val;
2324 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2326 #ifdef CONFIG_X86_64
2327 return ctxt->ops->guest_has_long_mode(ctxt);
2333 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2335 desc->g = (flags >> 23) & 1;
2336 desc->d = (flags >> 22) & 1;
2337 desc->l = (flags >> 21) & 1;
2338 desc->avl = (flags >> 20) & 1;
2339 desc->p = (flags >> 15) & 1;
2340 desc->dpl = (flags >> 13) & 3;
2341 desc->s = (flags >> 12) & 1;
2342 desc->type = (flags >> 8) & 15;
2345 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2348 struct desc_struct desc;
2352 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2355 offset = 0x7f84 + n * 12;
2357 offset = 0x7f2c + (n - 3) * 12;
2359 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2360 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2361 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2362 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2363 return X86EMUL_CONTINUE;
2366 #ifdef CONFIG_X86_64
2367 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2370 struct desc_struct desc;
2375 offset = 0x7e00 + n * 16;
2377 selector = GET_SMSTATE(u16, smstate, offset);
2378 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2379 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2380 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2381 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2383 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2384 return X86EMUL_CONTINUE;
2388 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2389 u64 cr0, u64 cr3, u64 cr4)
2394 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2396 if (cr4 & X86_CR4_PCIDE) {
2401 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2403 return X86EMUL_UNHANDLEABLE;
2406 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2407 * Then enable protected mode. However, PCID cannot be enabled
2408 * if EFER.LMA=0, so set it separately.
2410 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2412 return X86EMUL_UNHANDLEABLE;
2414 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2416 return X86EMUL_UNHANDLEABLE;
2418 if (cr4 & X86_CR4_PCIDE) {
2419 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2421 return X86EMUL_UNHANDLEABLE;
2423 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2425 return X86EMUL_UNHANDLEABLE;
2430 return X86EMUL_CONTINUE;
2433 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2434 const char *smstate)
2436 struct desc_struct desc;
2439 u32 val, cr0, cr3, cr4;
2442 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2443 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2444 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2445 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2447 for (i = 0; i < NR_EMULATOR_GPRS; i++)
2448 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2450 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2452 if (ctxt->ops->set_dr(ctxt, 6, val))
2453 return X86EMUL_UNHANDLEABLE;
2455 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2457 if (ctxt->ops->set_dr(ctxt, 7, val))
2458 return X86EMUL_UNHANDLEABLE;
2460 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2461 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2462 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2463 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2464 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2466 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2467 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2468 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2469 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2470 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2472 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2473 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2474 ctxt->ops->set_gdt(ctxt, &dt);
2476 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2477 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2478 ctxt->ops->set_idt(ctxt, &dt);
2480 for (i = 0; i < 6; i++) {
2481 int r = rsm_load_seg_32(ctxt, smstate, i);
2482 if (r != X86EMUL_CONTINUE)
2486 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2488 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2490 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2493 #ifdef CONFIG_X86_64
2494 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2495 const char *smstate)
2497 struct desc_struct desc;
2499 u64 val, cr0, cr3, cr4;
2504 for (i = 0; i < NR_EMULATOR_GPRS; i++)
2505 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2507 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2508 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2510 val = GET_SMSTATE(u64, smstate, 0x7f68);
2512 if (ctxt->ops->set_dr(ctxt, 6, val))
2513 return X86EMUL_UNHANDLEABLE;
2515 val = GET_SMSTATE(u64, smstate, 0x7f60);
2517 if (ctxt->ops->set_dr(ctxt, 7, val))
2518 return X86EMUL_UNHANDLEABLE;
2520 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2521 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2522 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2523 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2524 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2526 if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))
2527 return X86EMUL_UNHANDLEABLE;
2529 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2530 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2531 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2532 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2533 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2534 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2536 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2537 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2538 ctxt->ops->set_idt(ctxt, &dt);
2540 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2541 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2542 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2543 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2544 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2545 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2547 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2548 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2549 ctxt->ops->set_gdt(ctxt, &dt);
2551 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2552 if (r != X86EMUL_CONTINUE)
2555 for (i = 0; i < 6; i++) {
2556 r = rsm_load_seg_64(ctxt, smstate, i);
2557 if (r != X86EMUL_CONTINUE)
2561 return X86EMUL_CONTINUE;
2565 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2567 unsigned long cr0, cr4, efer;
2572 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2573 return emulate_ud(ctxt);
2575 smbase = ctxt->ops->get_smbase(ctxt);
2577 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2578 if (ret != X86EMUL_CONTINUE)
2579 return X86EMUL_UNHANDLEABLE;
2581 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2582 ctxt->ops->set_nmi_mask(ctxt, false);
2584 ctxt->ops->exiting_smm(ctxt);
2587 * Get back to real mode, to prepare a safe state in which to load
2588 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2589 * supports long mode.
2591 if (emulator_has_longmode(ctxt)) {
2592 struct desc_struct cs_desc;
2594 /* Zero CR4.PCIDE before CR0.PG. */
2595 cr4 = ctxt->ops->get_cr(ctxt, 4);
2596 if (cr4 & X86_CR4_PCIDE)
2597 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2599 /* A 32-bit code segment is required to clear EFER.LMA. */
2600 memset(&cs_desc, 0, sizeof(cs_desc));
2602 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2603 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2606 /* For the 64-bit case, this will clear EFER.LMA. */
2607 cr0 = ctxt->ops->get_cr(ctxt, 0);
2608 if (cr0 & X86_CR0_PE)
2609 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2611 if (emulator_has_longmode(ctxt)) {
2612 /* Clear CR4.PAE before clearing EFER.LME. */
2613 cr4 = ctxt->ops->get_cr(ctxt, 4);
2614 if (cr4 & X86_CR4_PAE)
2615 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2617 /* And finally go back to 32-bit mode. */
2619 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2623 * Give leave_smm() a chance to make ISA-specific changes to the vCPU
2624 * state (e.g. enter guest mode) before loading state from the SMM
2627 if (ctxt->ops->leave_smm(ctxt, buf))
2628 goto emulate_shutdown;
2630 #ifdef CONFIG_X86_64
2631 if (emulator_has_longmode(ctxt))
2632 ret = rsm_load_state_64(ctxt, buf);
2635 ret = rsm_load_state_32(ctxt, buf);
2637 if (ret != X86EMUL_CONTINUE)
2638 goto emulate_shutdown;
2641 * Note, the ctxt->ops callbacks are responsible for handling side
2642 * effects when writing MSRs and CRs, e.g. MMU context resets, CPUID
2643 * runtime updates, etc... If that changes, e.g. this flow is moved
2644 * out of the emulator to make it look more like enter_smm(), then
2645 * those side effects need to be explicitly handled for both success
2648 return X86EMUL_CONTINUE;
2651 ctxt->ops->triple_fault(ctxt);
2652 return X86EMUL_CONTINUE;
2656 setup_syscalls_segments(struct desc_struct *cs, struct desc_struct *ss)
2658 cs->l = 0; /* will be adjusted later */
2659 set_desc_base(cs, 0); /* flat segment */
2660 cs->g = 1; /* 4kb granularity */
2661 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2662 cs->type = 0x0b; /* Read, Execute, Accessed */
2664 cs->dpl = 0; /* will be adjusted later */
2669 set_desc_base(ss, 0); /* flat segment */
2670 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2671 ss->g = 1; /* 4kb granularity */
2673 ss->type = 0x03; /* Read/Write, Accessed */
2674 ss->d = 1; /* 32bit stack segment */
2681 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2683 u32 eax, ebx, ecx, edx;
2686 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2687 return is_guest_vendor_intel(ebx, ecx, edx);
2690 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2692 const struct x86_emulate_ops *ops = ctxt->ops;
2693 u32 eax, ebx, ecx, edx;
2696 * syscall should always be enabled in longmode - so only become
2697 * vendor specific (cpuid) if other modes are active...
2699 if (ctxt->mode == X86EMUL_MODE_PROT64)
2704 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2706 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
2707 * 64bit guest with a 32bit compat-app running will #UD !! While this
2708 * behaviour can be fixed (by emulating) into AMD response - CPUs of
2709 * AMD can't behave like Intel.
2711 if (is_guest_vendor_intel(ebx, ecx, edx))
2714 if (is_guest_vendor_amd(ebx, ecx, edx) ||
2715 is_guest_vendor_hygon(ebx, ecx, edx))
2719 * default: (not Intel, not AMD, not Hygon), apply Intel's
2725 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2727 const struct x86_emulate_ops *ops = ctxt->ops;
2728 struct desc_struct cs, ss;
2733 /* syscall is not available in real mode */
2734 if (ctxt->mode == X86EMUL_MODE_REAL ||
2735 ctxt->mode == X86EMUL_MODE_VM86)
2736 return emulate_ud(ctxt);
2738 if (!(em_syscall_is_enabled(ctxt)))
2739 return emulate_ud(ctxt);
2741 ops->get_msr(ctxt, MSR_EFER, &efer);
2742 if (!(efer & EFER_SCE))
2743 return emulate_ud(ctxt);
2745 setup_syscalls_segments(&cs, &ss);
2746 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2748 cs_sel = (u16)(msr_data & 0xfffc);
2749 ss_sel = (u16)(msr_data + 8);
2751 if (efer & EFER_LMA) {
2755 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2756 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2758 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2759 if (efer & EFER_LMA) {
2760 #ifdef CONFIG_X86_64
2761 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2764 ctxt->mode == X86EMUL_MODE_PROT64 ?
2765 MSR_LSTAR : MSR_CSTAR, &msr_data);
2766 ctxt->_eip = msr_data;
2768 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2769 ctxt->eflags &= ~msr_data;
2770 ctxt->eflags |= X86_EFLAGS_FIXED;
2774 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2775 ctxt->_eip = (u32)msr_data;
2777 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2780 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2781 return X86EMUL_CONTINUE;
2784 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2786 const struct x86_emulate_ops *ops = ctxt->ops;
2787 struct desc_struct cs, ss;
2792 ops->get_msr(ctxt, MSR_EFER, &efer);
2793 /* inject #GP if in real mode */
2794 if (ctxt->mode == X86EMUL_MODE_REAL)
2795 return emulate_gp(ctxt, 0);
2798 * Not recognized on AMD in compat mode (but is recognized in legacy
2801 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2802 && !vendor_intel(ctxt))
2803 return emulate_ud(ctxt);
2805 /* sysenter/sysexit have not been tested in 64bit mode. */
2806 if (ctxt->mode == X86EMUL_MODE_PROT64)
2807 return X86EMUL_UNHANDLEABLE;
2809 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2810 if ((msr_data & 0xfffc) == 0x0)
2811 return emulate_gp(ctxt, 0);
2813 setup_syscalls_segments(&cs, &ss);
2814 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2815 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2816 ss_sel = cs_sel + 8;
2817 if (efer & EFER_LMA) {
2822 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2823 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2825 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2826 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2828 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2829 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2831 if (efer & EFER_LMA)
2832 ctxt->mode = X86EMUL_MODE_PROT64;
2834 return X86EMUL_CONTINUE;
2837 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2839 const struct x86_emulate_ops *ops = ctxt->ops;
2840 struct desc_struct cs, ss;
2841 u64 msr_data, rcx, rdx;
2843 u16 cs_sel = 0, ss_sel = 0;
2845 /* inject #GP if in real mode or Virtual 8086 mode */
2846 if (ctxt->mode == X86EMUL_MODE_REAL ||
2847 ctxt->mode == X86EMUL_MODE_VM86)
2848 return emulate_gp(ctxt, 0);
2850 setup_syscalls_segments(&cs, &ss);
2852 if ((ctxt->rex_prefix & 0x8) != 0x0)
2853 usermode = X86EMUL_MODE_PROT64;
2855 usermode = X86EMUL_MODE_PROT32;
2857 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2858 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2862 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2864 case X86EMUL_MODE_PROT32:
2865 cs_sel = (u16)(msr_data + 16);
2866 if ((msr_data & 0xfffc) == 0x0)
2867 return emulate_gp(ctxt, 0);
2868 ss_sel = (u16)(msr_data + 24);
2872 case X86EMUL_MODE_PROT64:
2873 cs_sel = (u16)(msr_data + 32);
2874 if (msr_data == 0x0)
2875 return emulate_gp(ctxt, 0);
2876 ss_sel = cs_sel + 8;
2879 if (emul_is_noncanonical_address(rcx, ctxt) ||
2880 emul_is_noncanonical_address(rdx, ctxt))
2881 return emulate_gp(ctxt, 0);
2884 cs_sel |= SEGMENT_RPL_MASK;
2885 ss_sel |= SEGMENT_RPL_MASK;
2887 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2888 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2891 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2893 return X86EMUL_CONTINUE;
2896 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2899 if (ctxt->mode == X86EMUL_MODE_REAL)
2901 if (ctxt->mode == X86EMUL_MODE_VM86)
2903 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2904 return ctxt->ops->cpl(ctxt) > iopl;
2907 #define VMWARE_PORT_VMPORT (0x5658)
2908 #define VMWARE_PORT_VMRPC (0x5659)
2910 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2913 const struct x86_emulate_ops *ops = ctxt->ops;
2914 struct desc_struct tr_seg;
2917 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2918 unsigned mask = (1 << len) - 1;
2922 * VMware allows access to these ports even if denied
2923 * by TSS I/O permission bitmap. Mimic behavior.
2925 if (enable_vmware_backdoor &&
2926 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2929 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2932 if (desc_limit_scaled(&tr_seg) < 103)
2934 base = get_desc_base(&tr_seg);
2935 #ifdef CONFIG_X86_64
2936 base |= ((u64)base3) << 32;
2938 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2939 if (r != X86EMUL_CONTINUE)
2941 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2943 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2944 if (r != X86EMUL_CONTINUE)
2946 if ((perm >> bit_idx) & mask)
2951 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2957 if (emulator_bad_iopl(ctxt))
2958 if (!emulator_io_port_access_allowed(ctxt, port, len))
2961 ctxt->perm_ok = true;
2966 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2969 * Intel CPUs mask the counter and pointers in quite strange
2970 * manner when ECX is zero due to REP-string optimizations.
2972 #ifdef CONFIG_X86_64
2973 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2976 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2979 case 0xa4: /* movsb */
2980 case 0xa5: /* movsd/w */
2981 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2983 case 0xaa: /* stosb */
2984 case 0xab: /* stosd/w */
2985 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2990 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2991 struct tss_segment_16 *tss)
2993 tss->ip = ctxt->_eip;
2994 tss->flag = ctxt->eflags;
2995 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2996 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2997 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2998 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2999 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3000 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3001 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3002 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3004 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3005 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3006 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3007 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3008 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3011 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3012 struct tss_segment_16 *tss)
3017 ctxt->_eip = tss->ip;
3018 ctxt->eflags = tss->flag | 2;
3019 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3020 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3021 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3022 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3023 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3024 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3025 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3026 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3029 * SDM says that segment selectors are loaded before segment
3032 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3033 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3034 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3035 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3036 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3041 * Now load segment descriptors. If fault happens at this stage
3042 * it is handled in a context of new task
3044 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3045 X86_TRANSFER_TASK_SWITCH, NULL);
3046 if (ret != X86EMUL_CONTINUE)
3048 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3049 X86_TRANSFER_TASK_SWITCH, NULL);
3050 if (ret != X86EMUL_CONTINUE)
3052 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3053 X86_TRANSFER_TASK_SWITCH, NULL);
3054 if (ret != X86EMUL_CONTINUE)
3056 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3057 X86_TRANSFER_TASK_SWITCH, NULL);
3058 if (ret != X86EMUL_CONTINUE)
3060 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3061 X86_TRANSFER_TASK_SWITCH, NULL);
3062 if (ret != X86EMUL_CONTINUE)
3065 return X86EMUL_CONTINUE;
3068 static int task_switch_16(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3069 ulong old_tss_base, struct desc_struct *new_desc)
3071 struct tss_segment_16 tss_seg;
3073 u32 new_tss_base = get_desc_base(new_desc);
3075 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3076 if (ret != X86EMUL_CONTINUE)
3079 save_state_to_tss16(ctxt, &tss_seg);
3081 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3082 if (ret != X86EMUL_CONTINUE)
3085 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3086 if (ret != X86EMUL_CONTINUE)
3089 if (old_tss_sel != 0xffff) {
3090 tss_seg.prev_task_link = old_tss_sel;
3092 ret = linear_write_system(ctxt, new_tss_base,
3093 &tss_seg.prev_task_link,
3094 sizeof(tss_seg.prev_task_link));
3095 if (ret != X86EMUL_CONTINUE)
3099 return load_state_from_tss16(ctxt, &tss_seg);
3102 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3103 struct tss_segment_32 *tss)
3105 /* CR3 and ldt selector are not saved intentionally */
3106 tss->eip = ctxt->_eip;
3107 tss->eflags = ctxt->eflags;
3108 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3109 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3110 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3111 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3112 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3113 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3114 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3115 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3117 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3118 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3119 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3120 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3121 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3122 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3125 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3126 struct tss_segment_32 *tss)
3131 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3132 return emulate_gp(ctxt, 0);
3133 ctxt->_eip = tss->eip;
3134 ctxt->eflags = tss->eflags | 2;
3136 /* General purpose registers */
3137 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3138 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3139 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3140 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3141 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3142 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3143 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3144 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3147 * SDM says that segment selectors are loaded before segment
3148 * descriptors. This is important because CPL checks will
3151 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3152 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3153 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3154 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3155 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3156 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3157 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3160 * If we're switching between Protected Mode and VM86, we need to make
3161 * sure to update the mode before loading the segment descriptors so
3162 * that the selectors are interpreted correctly.
3164 if (ctxt->eflags & X86_EFLAGS_VM) {
3165 ctxt->mode = X86EMUL_MODE_VM86;
3168 ctxt->mode = X86EMUL_MODE_PROT32;
3173 * Now load segment descriptors. If fault happens at this stage
3174 * it is handled in a context of new task
3176 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3177 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3178 if (ret != X86EMUL_CONTINUE)
3180 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3181 X86_TRANSFER_TASK_SWITCH, NULL);
3182 if (ret != X86EMUL_CONTINUE)
3184 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3185 X86_TRANSFER_TASK_SWITCH, NULL);
3186 if (ret != X86EMUL_CONTINUE)
3188 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3189 X86_TRANSFER_TASK_SWITCH, NULL);
3190 if (ret != X86EMUL_CONTINUE)
3192 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3193 X86_TRANSFER_TASK_SWITCH, NULL);
3194 if (ret != X86EMUL_CONTINUE)
3196 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3197 X86_TRANSFER_TASK_SWITCH, NULL);
3198 if (ret != X86EMUL_CONTINUE)
3200 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3201 X86_TRANSFER_TASK_SWITCH, NULL);
3206 static int task_switch_32(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3207 ulong old_tss_base, struct desc_struct *new_desc)
3209 struct tss_segment_32 tss_seg;
3211 u32 new_tss_base = get_desc_base(new_desc);
3212 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3213 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3215 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3216 if (ret != X86EMUL_CONTINUE)
3219 save_state_to_tss32(ctxt, &tss_seg);
3221 /* Only GP registers and segment selectors are saved */
3222 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3223 ldt_sel_offset - eip_offset);
3224 if (ret != X86EMUL_CONTINUE)
3227 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3228 if (ret != X86EMUL_CONTINUE)
3231 if (old_tss_sel != 0xffff) {
3232 tss_seg.prev_task_link = old_tss_sel;
3234 ret = linear_write_system(ctxt, new_tss_base,
3235 &tss_seg.prev_task_link,
3236 sizeof(tss_seg.prev_task_link));
3237 if (ret != X86EMUL_CONTINUE)
3241 return load_state_from_tss32(ctxt, &tss_seg);
3244 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3245 u16 tss_selector, int idt_index, int reason,
3246 bool has_error_code, u32 error_code)
3248 const struct x86_emulate_ops *ops = ctxt->ops;
3249 struct desc_struct curr_tss_desc, next_tss_desc;
3251 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3252 ulong old_tss_base =
3253 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3255 ulong desc_addr, dr7;
3257 /* FIXME: old_tss_base == ~0 ? */
3259 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3260 if (ret != X86EMUL_CONTINUE)
3262 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3263 if (ret != X86EMUL_CONTINUE)
3266 /* FIXME: check that next_tss_desc is tss */
3269 * Check privileges. The three cases are task switch caused by...
3271 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3272 * 2. Exception/IRQ/iret: No check is performed
3273 * 3. jmp/call to TSS/task-gate: No check is performed since the
3274 * hardware checks it before exiting.
3276 if (reason == TASK_SWITCH_GATE) {
3277 if (idt_index != -1) {
3278 /* Software interrupts */
3279 struct desc_struct task_gate_desc;
3282 ret = read_interrupt_descriptor(ctxt, idt_index,
3284 if (ret != X86EMUL_CONTINUE)
3287 dpl = task_gate_desc.dpl;
3288 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3289 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3293 desc_limit = desc_limit_scaled(&next_tss_desc);
3294 if (!next_tss_desc.p ||
3295 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3296 desc_limit < 0x2b)) {
3297 return emulate_ts(ctxt, tss_selector & 0xfffc);
3300 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3301 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3302 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3305 if (reason == TASK_SWITCH_IRET)
3306 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3308 /* set back link to prev task only if NT bit is set in eflags
3309 note that old_tss_sel is not used after this point */
3310 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3311 old_tss_sel = 0xffff;
3313 if (next_tss_desc.type & 8)
3314 ret = task_switch_32(ctxt, old_tss_sel, old_tss_base, &next_tss_desc);
3316 ret = task_switch_16(ctxt, old_tss_sel,
3317 old_tss_base, &next_tss_desc);
3318 if (ret != X86EMUL_CONTINUE)
3321 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3322 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3324 if (reason != TASK_SWITCH_IRET) {
3325 next_tss_desc.type |= (1 << 1); /* set busy flag */
3326 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3329 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3330 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3332 if (has_error_code) {
3333 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3334 ctxt->lock_prefix = 0;
3335 ctxt->src.val = (unsigned long) error_code;
3336 ret = em_push(ctxt);
3339 ops->get_dr(ctxt, 7, &dr7);
3340 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3345 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3346 u16 tss_selector, int idt_index, int reason,
3347 bool has_error_code, u32 error_code)
3351 invalidate_registers(ctxt);
3352 ctxt->_eip = ctxt->eip;
3353 ctxt->dst.type = OP_NONE;
3355 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3356 has_error_code, error_code);
3358 if (rc == X86EMUL_CONTINUE) {
3359 ctxt->eip = ctxt->_eip;
3360 writeback_registers(ctxt);
3363 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3366 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3369 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3371 register_address_increment(ctxt, reg, df * op->bytes);
3372 op->addr.mem.ea = register_address(ctxt, reg);
3375 static int em_das(struct x86_emulate_ctxt *ctxt)
3378 bool af, cf, old_cf;
3380 cf = ctxt->eflags & X86_EFLAGS_CF;
3386 af = ctxt->eflags & X86_EFLAGS_AF;
3387 if ((al & 0x0f) > 9 || af) {
3389 cf = old_cf | (al >= 250);
3394 if (old_al > 0x99 || old_cf) {
3400 /* Set PF, ZF, SF */
3401 ctxt->src.type = OP_IMM;
3403 ctxt->src.bytes = 1;
3404 fastop(ctxt, em_or);
3405 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3407 ctxt->eflags |= X86_EFLAGS_CF;
3409 ctxt->eflags |= X86_EFLAGS_AF;
3410 return X86EMUL_CONTINUE;
3413 static int em_aam(struct x86_emulate_ctxt *ctxt)
3417 if (ctxt->src.val == 0)
3418 return emulate_de(ctxt);
3420 al = ctxt->dst.val & 0xff;
3421 ah = al / ctxt->src.val;
3422 al %= ctxt->src.val;
3424 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3426 /* Set PF, ZF, SF */
3427 ctxt->src.type = OP_IMM;
3429 ctxt->src.bytes = 1;
3430 fastop(ctxt, em_or);
3432 return X86EMUL_CONTINUE;
3435 static int em_aad(struct x86_emulate_ctxt *ctxt)
3437 u8 al = ctxt->dst.val & 0xff;
3438 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3440 al = (al + (ah * ctxt->src.val)) & 0xff;
3442 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3444 /* Set PF, ZF, SF */
3445 ctxt->src.type = OP_IMM;
3447 ctxt->src.bytes = 1;
3448 fastop(ctxt, em_or);
3450 return X86EMUL_CONTINUE;
3453 static int em_call(struct x86_emulate_ctxt *ctxt)
3456 long rel = ctxt->src.val;
3458 ctxt->src.val = (unsigned long)ctxt->_eip;
3459 rc = jmp_rel(ctxt, rel);
3460 if (rc != X86EMUL_CONTINUE)
3462 return em_push(ctxt);
3465 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3470 struct desc_struct old_desc, new_desc;
3471 const struct x86_emulate_ops *ops = ctxt->ops;
3472 int cpl = ctxt->ops->cpl(ctxt);
3473 enum x86emul_mode prev_mode = ctxt->mode;
3475 old_eip = ctxt->_eip;
3476 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3478 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3479 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3480 X86_TRANSFER_CALL_JMP, &new_desc);
3481 if (rc != X86EMUL_CONTINUE)
3484 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3485 if (rc != X86EMUL_CONTINUE)
3488 ctxt->src.val = old_cs;
3490 if (rc != X86EMUL_CONTINUE)
3493 ctxt->src.val = old_eip;
3495 /* If we failed, we tainted the memory, but the very least we should
3497 if (rc != X86EMUL_CONTINUE) {
3498 pr_warn_once("faulting far call emulation tainted memory\n");
3503 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3504 ctxt->mode = prev_mode;
3509 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3514 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3515 if (rc != X86EMUL_CONTINUE)
3517 rc = assign_eip_near(ctxt, eip);
3518 if (rc != X86EMUL_CONTINUE)
3520 rsp_increment(ctxt, ctxt->src.val);
3521 return X86EMUL_CONTINUE;
3524 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3526 /* Write back the register source. */
3527 ctxt->src.val = ctxt->dst.val;
3528 write_register_operand(&ctxt->src);
3530 /* Write back the memory destination with implicit LOCK prefix. */
3531 ctxt->dst.val = ctxt->src.orig_val;
3532 ctxt->lock_prefix = 1;
3533 return X86EMUL_CONTINUE;
3536 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3538 ctxt->dst.val = ctxt->src2.val;
3539 return fastop(ctxt, em_imul);
3542 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3544 ctxt->dst.type = OP_REG;
3545 ctxt->dst.bytes = ctxt->src.bytes;
3546 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3547 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3549 return X86EMUL_CONTINUE;
3552 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3556 if (!ctxt->ops->guest_has_rdpid(ctxt))
3557 return emulate_ud(ctxt);
3559 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);
3560 ctxt->dst.val = tsc_aux;
3561 return X86EMUL_CONTINUE;
3564 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3568 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3569 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3570 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3571 return X86EMUL_CONTINUE;
3574 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3578 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3579 return emulate_gp(ctxt, 0);
3580 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3581 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3582 return X86EMUL_CONTINUE;
3585 static int em_mov(struct x86_emulate_ctxt *ctxt)
3587 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3588 return X86EMUL_CONTINUE;
3591 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3595 if (!ctxt->ops->guest_has_movbe(ctxt))
3596 return emulate_ud(ctxt);
3598 switch (ctxt->op_bytes) {
3601 * From MOVBE definition: "...When the operand size is 16 bits,
3602 * the upper word of the destination register remains unchanged
3605 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3606 * rules so we have to do the operation almost per hand.
3608 tmp = (u16)ctxt->src.val;
3609 ctxt->dst.val &= ~0xffffUL;
3610 ctxt->dst.val |= (unsigned long)swab16(tmp);
3613 ctxt->dst.val = swab32((u32)ctxt->src.val);
3616 ctxt->dst.val = swab64(ctxt->src.val);
3621 return X86EMUL_CONTINUE;
3624 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3626 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3627 return emulate_gp(ctxt, 0);
3629 /* Disable writeback. */
3630 ctxt->dst.type = OP_NONE;
3631 return X86EMUL_CONTINUE;
3634 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3638 if (ctxt->mode == X86EMUL_MODE_PROT64)
3639 val = ctxt->src.val & ~0ULL;
3641 val = ctxt->src.val & ~0U;
3643 /* #UD condition is already handled. */
3644 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3645 return emulate_gp(ctxt, 0);
3647 /* Disable writeback. */
3648 ctxt->dst.type = OP_NONE;
3649 return X86EMUL_CONTINUE;
3652 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3654 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3658 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3659 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3660 r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data);
3662 if (r == X86EMUL_IO_NEEDED)
3666 return emulate_gp(ctxt, 0);
3668 return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
3671 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3673 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3677 r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data);
3679 if (r == X86EMUL_IO_NEEDED)
3683 return emulate_gp(ctxt, 0);
3685 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3686 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3687 return X86EMUL_CONTINUE;
3690 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3692 if (segment > VCPU_SREG_GS &&
3693 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3694 ctxt->ops->cpl(ctxt) > 0)
3695 return emulate_gp(ctxt, 0);
3697 ctxt->dst.val = get_segment_selector(ctxt, segment);
3698 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3699 ctxt->dst.bytes = 2;
3700 return X86EMUL_CONTINUE;
3703 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3705 if (ctxt->modrm_reg > VCPU_SREG_GS)
3706 return emulate_ud(ctxt);
3708 return em_store_sreg(ctxt, ctxt->modrm_reg);
3711 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3713 u16 sel = ctxt->src.val;
3715 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3716 return emulate_ud(ctxt);
3718 if (ctxt->modrm_reg == VCPU_SREG_SS)
3719 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3721 /* Disable writeback. */
3722 ctxt->dst.type = OP_NONE;
3723 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3726 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3728 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3731 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3733 u16 sel = ctxt->src.val;
3735 /* Disable writeback. */
3736 ctxt->dst.type = OP_NONE;
3737 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3740 static int em_str(struct x86_emulate_ctxt *ctxt)
3742 return em_store_sreg(ctxt, VCPU_SREG_TR);
3745 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3747 u16 sel = ctxt->src.val;
3749 /* Disable writeback. */
3750 ctxt->dst.type = OP_NONE;
3751 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3754 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3759 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3760 if (rc == X86EMUL_CONTINUE)
3761 ctxt->ops->invlpg(ctxt, linear);
3762 /* Disable writeback. */
3763 ctxt->dst.type = OP_NONE;
3764 return X86EMUL_CONTINUE;
3767 static int em_clts(struct x86_emulate_ctxt *ctxt)
3771 cr0 = ctxt->ops->get_cr(ctxt, 0);
3773 ctxt->ops->set_cr(ctxt, 0, cr0);
3774 return X86EMUL_CONTINUE;
3777 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3779 int rc = ctxt->ops->fix_hypercall(ctxt);
3781 if (rc != X86EMUL_CONTINUE)
3784 /* Let the processor re-execute the fixed hypercall */
3785 ctxt->_eip = ctxt->eip;
3786 /* Disable writeback. */
3787 ctxt->dst.type = OP_NONE;
3788 return X86EMUL_CONTINUE;
3791 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3792 void (*get)(struct x86_emulate_ctxt *ctxt,
3793 struct desc_ptr *ptr))
3795 struct desc_ptr desc_ptr;
3797 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3798 ctxt->ops->cpl(ctxt) > 0)
3799 return emulate_gp(ctxt, 0);
3801 if (ctxt->mode == X86EMUL_MODE_PROT64)
3803 get(ctxt, &desc_ptr);
3804 if (ctxt->op_bytes == 2) {
3806 desc_ptr.address &= 0x00ffffff;
3808 /* Disable writeback. */
3809 ctxt->dst.type = OP_NONE;
3810 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3811 &desc_ptr, 2 + ctxt->op_bytes);
3814 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3816 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3819 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3821 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3824 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3826 struct desc_ptr desc_ptr;
3829 if (ctxt->mode == X86EMUL_MODE_PROT64)
3831 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3832 &desc_ptr.size, &desc_ptr.address,
3834 if (rc != X86EMUL_CONTINUE)
3836 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3837 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3838 return emulate_gp(ctxt, 0);
3840 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3842 ctxt->ops->set_idt(ctxt, &desc_ptr);
3843 /* Disable writeback. */
3844 ctxt->dst.type = OP_NONE;
3845 return X86EMUL_CONTINUE;
3848 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3850 return em_lgdt_lidt(ctxt, true);
3853 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3855 return em_lgdt_lidt(ctxt, false);
3858 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3860 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3861 ctxt->ops->cpl(ctxt) > 0)
3862 return emulate_gp(ctxt, 0);
3864 if (ctxt->dst.type == OP_MEM)
3865 ctxt->dst.bytes = 2;
3866 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3867 return X86EMUL_CONTINUE;
3870 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3872 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3873 | (ctxt->src.val & 0x0f));
3874 ctxt->dst.type = OP_NONE;
3875 return X86EMUL_CONTINUE;
3878 static int em_loop(struct x86_emulate_ctxt *ctxt)
3880 int rc = X86EMUL_CONTINUE;
3882 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3883 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3884 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3885 rc = jmp_rel(ctxt, ctxt->src.val);
3890 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3892 int rc = X86EMUL_CONTINUE;
3894 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3895 rc = jmp_rel(ctxt, ctxt->src.val);
3900 static int em_in(struct x86_emulate_ctxt *ctxt)
3902 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3904 return X86EMUL_IO_NEEDED;
3906 return X86EMUL_CONTINUE;
3909 static int em_out(struct x86_emulate_ctxt *ctxt)
3911 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3913 /* Disable writeback. */
3914 ctxt->dst.type = OP_NONE;
3915 return X86EMUL_CONTINUE;
3918 static int em_cli(struct x86_emulate_ctxt *ctxt)
3920 if (emulator_bad_iopl(ctxt))
3921 return emulate_gp(ctxt, 0);
3923 ctxt->eflags &= ~X86_EFLAGS_IF;
3924 return X86EMUL_CONTINUE;
3927 static int em_sti(struct x86_emulate_ctxt *ctxt)
3929 if (emulator_bad_iopl(ctxt))
3930 return emulate_gp(ctxt, 0);
3932 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3933 ctxt->eflags |= X86_EFLAGS_IF;
3934 return X86EMUL_CONTINUE;
3937 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3939 u32 eax, ebx, ecx, edx;
3942 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3943 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3944 ctxt->ops->cpl(ctxt)) {
3945 return emulate_gp(ctxt, 0);
3948 eax = reg_read(ctxt, VCPU_REGS_RAX);
3949 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3950 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3951 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3952 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3953 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3954 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3955 return X86EMUL_CONTINUE;
3958 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3962 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3964 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3966 ctxt->eflags &= ~0xffUL;
3967 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3968 return X86EMUL_CONTINUE;
3971 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3973 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3974 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3975 return X86EMUL_CONTINUE;
3978 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3980 switch (ctxt->op_bytes) {
3981 #ifdef CONFIG_X86_64
3983 asm("bswap %0" : "+r"(ctxt->dst.val));
3987 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3990 return X86EMUL_CONTINUE;
3993 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3995 /* emulating clflush regardless of cpuid */
3996 return X86EMUL_CONTINUE;
3999 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
4001 /* emulating clflushopt regardless of cpuid */
4002 return X86EMUL_CONTINUE;
4005 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4007 ctxt->dst.val = (s32) ctxt->src.val;
4008 return X86EMUL_CONTINUE;
4011 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4013 if (!ctxt->ops->guest_has_fxsr(ctxt))
4014 return emulate_ud(ctxt);
4016 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4017 return emulate_nm(ctxt);
4020 * Don't emulate a case that should never be hit, instead of working
4021 * around a lack of fxsave64/fxrstor64 on old compilers.
4023 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4024 return X86EMUL_UNHANDLEABLE;
4026 return X86EMUL_CONTINUE;
4030 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4031 * and restore MXCSR.
4033 static size_t __fxstate_size(int nregs)
4035 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4038 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4041 if (ctxt->mode == X86EMUL_MODE_PROT64)
4042 return __fxstate_size(16);
4044 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4045 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4049 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4052 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4053 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4055 * 3) 64-bit mode with REX.W prefix
4056 * - like (2), but XMM 8-15 are being saved and restored
4057 * 4) 64-bit mode without REX.W prefix
4058 * - like (3), but FIP and FDP are 64 bit
4060 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4061 * desired result. (4) is not emulated.
4063 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4064 * and FPU DS) should match.
4066 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4068 struct fxregs_state fx_state;
4071 rc = check_fxsr(ctxt);
4072 if (rc != X86EMUL_CONTINUE)
4077 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4081 if (rc != X86EMUL_CONTINUE)
4084 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4085 fxstate_size(ctxt));
4089 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4090 * in the host registers (via FXSAVE) instead, so they won't be modified.
4091 * (preemption has to stay disabled until FXRSTOR).
4093 * Use noinline to keep the stack for other functions called by callers small.
4095 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4096 const size_t used_size)
4098 struct fxregs_state fx_tmp;
4101 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4102 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4103 __fxstate_size(16) - used_size);
4108 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4110 struct fxregs_state fx_state;
4114 rc = check_fxsr(ctxt);
4115 if (rc != X86EMUL_CONTINUE)
4118 size = fxstate_size(ctxt);
4119 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4120 if (rc != X86EMUL_CONTINUE)
4125 if (size < __fxstate_size(16)) {
4126 rc = fxregs_fixup(&fx_state, size);
4127 if (rc != X86EMUL_CONTINUE)
4131 if (fx_state.mxcsr >> 16) {
4132 rc = emulate_gp(ctxt, 0);
4136 if (rc == X86EMUL_CONTINUE)
4137 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4145 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4149 eax = reg_read(ctxt, VCPU_REGS_RAX);
4150 edx = reg_read(ctxt, VCPU_REGS_RDX);
4151 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4153 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4154 return emulate_gp(ctxt, 0);
4156 return X86EMUL_CONTINUE;
4159 static bool valid_cr(int nr)
4171 static int check_cr_access(struct x86_emulate_ctxt *ctxt)
4173 if (!valid_cr(ctxt->modrm_reg))
4174 return emulate_ud(ctxt);
4176 return X86EMUL_CONTINUE;
4179 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4183 ctxt->ops->get_dr(ctxt, 7, &dr7);
4185 /* Check if DR7.Global_Enable is set */
4186 return dr7 & (1 << 13);
4189 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4191 int dr = ctxt->modrm_reg;
4195 return emulate_ud(ctxt);
4197 cr4 = ctxt->ops->get_cr(ctxt, 4);
4198 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4199 return emulate_ud(ctxt);
4201 if (check_dr7_gd(ctxt)) {
4204 ctxt->ops->get_dr(ctxt, 6, &dr6);
4205 dr6 &= ~DR_TRAP_BITS;
4206 dr6 |= DR6_BD | DR6_ACTIVE_LOW;
4207 ctxt->ops->set_dr(ctxt, 6, dr6);
4208 return emulate_db(ctxt);
4211 return X86EMUL_CONTINUE;
4214 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4216 u64 new_val = ctxt->src.val64;
4217 int dr = ctxt->modrm_reg;
4219 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4220 return emulate_gp(ctxt, 0);
4222 return check_dr_read(ctxt);
4225 static int check_svme(struct x86_emulate_ctxt *ctxt)
4229 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4231 if (!(efer & EFER_SVME))
4232 return emulate_ud(ctxt);
4234 return X86EMUL_CONTINUE;
4237 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4239 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4241 /* Valid physical address? */
4242 if (rax & 0xffff000000000000ULL)
4243 return emulate_gp(ctxt, 0);
4245 return check_svme(ctxt);
4248 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4250 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4252 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4253 return emulate_gp(ctxt, 0);
4255 return X86EMUL_CONTINUE;
4258 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4260 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4261 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4264 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4265 * in Ring3 when CR4.PCE=0.
4267 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4268 return X86EMUL_CONTINUE;
4271 * If CR4.PCE is set, the SDM requires CPL=0 or CR0.PE=0. The CR0.PE
4272 * check however is unnecessary because CPL is always 0 outside
4275 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4276 ctxt->ops->check_pmc(ctxt, rcx))
4277 return emulate_gp(ctxt, 0);
4279 return X86EMUL_CONTINUE;
4282 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4284 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4285 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4286 return emulate_gp(ctxt, 0);
4288 return X86EMUL_CONTINUE;
4291 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4293 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4294 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4295 return emulate_gp(ctxt, 0);
4297 return X86EMUL_CONTINUE;
4300 #define D(_y) { .flags = (_y) }
4301 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4302 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4303 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4304 #define N D(NotImpl)
4305 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4306 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4307 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4308 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4309 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4310 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4311 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4312 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4313 #define II(_f, _e, _i) \
4314 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4315 #define IIP(_f, _e, _i, _p) \
4316 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4317 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4318 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4320 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4321 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4322 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4323 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4324 #define I2bvIP(_f, _e, _i, _p) \
4325 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4327 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4328 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4329 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4331 static const struct opcode group7_rm0[] = {
4333 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4337 static const struct opcode group7_rm1[] = {
4338 DI(SrcNone | Priv, monitor),
4339 DI(SrcNone | Priv, mwait),
4343 static const struct opcode group7_rm2[] = {
4345 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4349 static const struct opcode group7_rm3[] = {
4350 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4351 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4352 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4353 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4354 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4355 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4356 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4357 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4360 static const struct opcode group7_rm7[] = {
4362 DIP(SrcNone, rdtscp, check_rdtsc),
4366 static const struct opcode group1[] = {
4368 F(Lock | PageTable, em_or),
4371 F(Lock | PageTable, em_and),
4377 static const struct opcode group1A[] = {
4378 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4381 static const struct opcode group2[] = {
4382 F(DstMem | ModRM, em_rol),
4383 F(DstMem | ModRM, em_ror),
4384 F(DstMem | ModRM, em_rcl),
4385 F(DstMem | ModRM, em_rcr),
4386 F(DstMem | ModRM, em_shl),
4387 F(DstMem | ModRM, em_shr),
4388 F(DstMem | ModRM, em_shl),
4389 F(DstMem | ModRM, em_sar),
4392 static const struct opcode group3[] = {
4393 F(DstMem | SrcImm | NoWrite, em_test),
4394 F(DstMem | SrcImm | NoWrite, em_test),
4395 F(DstMem | SrcNone | Lock, em_not),
4396 F(DstMem | SrcNone | Lock, em_neg),
4397 F(DstXacc | Src2Mem, em_mul_ex),
4398 F(DstXacc | Src2Mem, em_imul_ex),
4399 F(DstXacc | Src2Mem, em_div_ex),
4400 F(DstXacc | Src2Mem, em_idiv_ex),
4403 static const struct opcode group4[] = {
4404 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4405 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4409 static const struct opcode group5[] = {
4410 F(DstMem | SrcNone | Lock, em_inc),
4411 F(DstMem | SrcNone | Lock, em_dec),
4412 I(SrcMem | NearBranch | IsBranch, em_call_near_abs),
4413 I(SrcMemFAddr | ImplicitOps | IsBranch, em_call_far),
4414 I(SrcMem | NearBranch | IsBranch, em_jmp_abs),
4415 I(SrcMemFAddr | ImplicitOps | IsBranch, em_jmp_far),
4416 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4419 static const struct opcode group6[] = {
4420 II(Prot | DstMem, em_sldt, sldt),
4421 II(Prot | DstMem, em_str, str),
4422 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4423 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4427 static const struct group_dual group7 = { {
4428 II(Mov | DstMem, em_sgdt, sgdt),
4429 II(Mov | DstMem, em_sidt, sidt),
4430 II(SrcMem | Priv, em_lgdt, lgdt),
4431 II(SrcMem | Priv, em_lidt, lidt),
4432 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4433 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4434 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4440 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4441 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4445 static const struct opcode group8[] = {
4447 F(DstMem | SrcImmByte | NoWrite, em_bt),
4448 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4449 F(DstMem | SrcImmByte | Lock, em_btr),
4450 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4454 * The "memory" destination is actually always a register, since we come
4455 * from the register case of group9.
4457 static const struct gprefix pfx_0f_c7_7 = {
4458 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdpid),
4462 static const struct group_dual group9 = { {
4463 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4465 N, N, N, N, N, N, N,
4466 GP(0, &pfx_0f_c7_7),
4469 static const struct opcode group11[] = {
4470 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4474 static const struct gprefix pfx_0f_ae_7 = {
4475 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4478 static const struct group_dual group15 = { {
4479 I(ModRM | Aligned16, em_fxsave),
4480 I(ModRM | Aligned16, em_fxrstor),
4481 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4483 N, N, N, N, N, N, N, N,
4486 static const struct gprefix pfx_0f_6f_0f_7f = {
4487 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4490 static const struct instr_dual instr_dual_0f_2b = {
4494 static const struct gprefix pfx_0f_2b = {
4495 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4498 static const struct gprefix pfx_0f_10_0f_11 = {
4499 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4502 static const struct gprefix pfx_0f_28_0f_29 = {
4503 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4506 static const struct gprefix pfx_0f_e7 = {
4507 N, I(Sse, em_mov), N, N,
4510 static const struct escape escape_d9 = { {
4511 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4514 N, N, N, N, N, N, N, N,
4516 N, N, N, N, N, N, N, N,
4518 N, N, N, N, N, N, N, N,
4520 N, N, N, N, N, N, N, N,
4522 N, N, N, N, N, N, N, N,
4524 N, N, N, N, N, N, N, N,
4526 N, N, N, N, N, N, N, N,
4528 N, N, N, N, N, N, N, N,
4531 static const struct escape escape_db = { {
4532 N, N, N, N, N, N, N, N,
4535 N, N, N, N, N, N, N, N,
4537 N, N, N, N, N, N, N, N,
4539 N, N, N, N, N, N, N, N,
4541 N, N, N, N, N, N, N, N,
4543 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4545 N, N, N, N, N, N, N, N,
4547 N, N, N, N, N, N, N, N,
4549 N, N, N, N, N, N, N, N,
4552 static const struct escape escape_dd = { {
4553 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4556 N, N, N, N, N, N, N, N,
4558 N, N, N, N, N, N, N, N,
4560 N, N, N, N, N, N, N, N,
4562 N, N, N, N, N, N, N, N,
4564 N, N, N, N, N, N, N, N,
4566 N, N, N, N, N, N, N, N,
4568 N, N, N, N, N, N, N, N,
4570 N, N, N, N, N, N, N, N,
4573 static const struct instr_dual instr_dual_0f_c3 = {
4574 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4577 static const struct mode_dual mode_dual_63 = {
4578 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4581 static const struct opcode opcode_table[256] = {
4583 F6ALU(Lock, em_add),
4584 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4585 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4587 F6ALU(Lock | PageTable, em_or),
4588 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4591 F6ALU(Lock, em_adc),
4592 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4593 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4595 F6ALU(Lock, em_sbb),
4596 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4597 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4599 F6ALU(Lock | PageTable, em_and), N, N,
4601 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4603 F6ALU(Lock, em_xor), N, N,
4605 F6ALU(NoWrite, em_cmp), N, N,
4607 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4609 X8(I(SrcReg | Stack, em_push)),
4611 X8(I(DstReg | Stack, em_pop)),
4613 I(ImplicitOps | Stack | No64, em_pusha),
4614 I(ImplicitOps | Stack | No64, em_popa),
4615 N, MD(ModRM, &mode_dual_63),
4618 I(SrcImm | Mov | Stack, em_push),
4619 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4620 I(SrcImmByte | Mov | Stack, em_push),
4621 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4622 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4623 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4625 X16(D(SrcImmByte | NearBranch | IsBranch)),
4627 G(ByteOp | DstMem | SrcImm, group1),
4628 G(DstMem | SrcImm, group1),
4629 G(ByteOp | DstMem | SrcImm | No64, group1),
4630 G(DstMem | SrcImmByte, group1),
4631 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4632 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4634 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4635 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4636 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4637 D(ModRM | SrcMem | NoAccess | DstReg),
4638 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4641 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4643 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4644 I(SrcImmFAddr | No64 | IsBranch, em_call_far), N,
4645 II(ImplicitOps | Stack, em_pushf, pushf),
4646 II(ImplicitOps | Stack, em_popf, popf),
4647 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4649 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4650 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4651 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4652 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4654 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4655 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4656 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4657 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4659 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4661 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4663 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4664 I(ImplicitOps | NearBranch | SrcImmU16 | IsBranch, em_ret_near_imm),
4665 I(ImplicitOps | NearBranch | IsBranch, em_ret),
4666 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4667 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4668 G(ByteOp, group11), G(0, group11),
4670 I(Stack | SrcImmU16 | Src2ImmByte | IsBranch, em_enter),
4671 I(Stack | IsBranch, em_leave),
4672 I(ImplicitOps | SrcImmU16 | IsBranch, em_ret_far_imm),
4673 I(ImplicitOps | IsBranch, em_ret_far),
4674 D(ImplicitOps | IsBranch), DI(SrcImmByte | IsBranch, intn),
4675 D(ImplicitOps | No64 | IsBranch),
4676 II(ImplicitOps | IsBranch, em_iret, iret),
4678 G(Src2One | ByteOp, group2), G(Src2One, group2),
4679 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4680 I(DstAcc | SrcImmUByte | No64, em_aam),
4681 I(DstAcc | SrcImmUByte | No64, em_aad),
4682 F(DstAcc | ByteOp | No64, em_salc),
4683 I(DstAcc | SrcXLat | ByteOp, em_mov),
4685 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4687 X3(I(SrcImmByte | NearBranch | IsBranch, em_loop)),
4688 I(SrcImmByte | NearBranch | IsBranch, em_jcxz),
4689 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4690 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4692 I(SrcImm | NearBranch | IsBranch, em_call),
4693 D(SrcImm | ImplicitOps | NearBranch | IsBranch),
4694 I(SrcImmFAddr | No64 | IsBranch, em_jmp_far),
4695 D(SrcImmByte | ImplicitOps | NearBranch | IsBranch),
4696 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4697 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4699 N, DI(ImplicitOps, icebp), N, N,
4700 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4701 G(ByteOp, group3), G(0, group3),
4703 D(ImplicitOps), D(ImplicitOps),
4704 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4705 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4708 static const struct opcode twobyte_table[256] = {
4710 G(0, group6), GD(0, &group7), N, N,
4711 N, I(ImplicitOps | EmulateOnUD | IsBranch, em_syscall),
4712 II(ImplicitOps | Priv, em_clts, clts), N,
4713 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4714 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4716 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4717 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4719 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4720 D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4721 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4722 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4723 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4724 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
4726 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access),
4727 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4728 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4730 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4733 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4734 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4735 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4738 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4739 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4740 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4741 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4742 I(ImplicitOps | EmulateOnUD | IsBranch, em_sysenter),
4743 I(ImplicitOps | Priv | EmulateOnUD | IsBranch, em_sysexit),
4745 N, N, N, N, N, N, N, N,
4747 X16(D(DstReg | SrcMem | ModRM)),
4749 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4754 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4759 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4761 X16(D(SrcImm | NearBranch | IsBranch)),
4763 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4765 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4766 II(ImplicitOps, em_cpuid, cpuid),
4767 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4768 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4769 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4771 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4772 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4773 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4774 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4775 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4776 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4778 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4779 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4780 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4781 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4782 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4783 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4787 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4788 I(DstReg | SrcMem | ModRM, em_bsf_c),
4789 I(DstReg | SrcMem | ModRM, em_bsr_c),
4790 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4792 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4793 N, ID(0, &instr_dual_0f_c3),
4794 N, N, N, GD(0, &group9),
4796 X8(I(DstReg, em_bswap)),
4798 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4800 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4801 N, N, N, N, N, N, N, N,
4803 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4806 static const struct instr_dual instr_dual_0f_38_f0 = {
4807 I(DstReg | SrcMem | Mov, em_movbe), N
4810 static const struct instr_dual instr_dual_0f_38_f1 = {
4811 I(DstMem | SrcReg | Mov, em_movbe), N
4814 static const struct gprefix three_byte_0f_38_f0 = {
4815 ID(0, &instr_dual_0f_38_f0), N, N, N
4818 static const struct gprefix three_byte_0f_38_f1 = {
4819 ID(0, &instr_dual_0f_38_f1), N, N, N
4823 * Insns below are selected by the prefix which indexed by the third opcode
4826 static const struct opcode opcode_map_0f_38[256] = {
4828 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4830 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4832 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4833 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4854 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4858 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4864 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4865 unsigned size, bool sign_extension)
4867 int rc = X86EMUL_CONTINUE;
4871 op->addr.mem.ea = ctxt->_eip;
4872 /* NB. Immediates are sign-extended as necessary. */
4873 switch (op->bytes) {
4875 op->val = insn_fetch(s8, ctxt);
4878 op->val = insn_fetch(s16, ctxt);
4881 op->val = insn_fetch(s32, ctxt);
4884 op->val = insn_fetch(s64, ctxt);
4887 if (!sign_extension) {
4888 switch (op->bytes) {
4896 op->val &= 0xffffffff;
4904 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4907 int rc = X86EMUL_CONTINUE;
4911 decode_register_operand(ctxt, op);
4914 rc = decode_imm(ctxt, op, 1, false);
4917 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4921 if (ctxt->d & BitOp)
4922 fetch_bit_operand(ctxt);
4923 op->orig_val = op->val;
4926 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4930 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4931 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4932 fetch_register_operand(op);
4933 op->orig_val = op->val;
4937 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4938 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4939 fetch_register_operand(op);
4940 op->orig_val = op->val;
4943 if (ctxt->d & ByteOp) {
4948 op->bytes = ctxt->op_bytes;
4949 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4950 fetch_register_operand(op);
4951 op->orig_val = op->val;
4955 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4957 register_address(ctxt, VCPU_REGS_RDI);
4958 op->addr.mem.seg = VCPU_SREG_ES;
4965 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4966 fetch_register_operand(op);
4971 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4974 rc = decode_imm(ctxt, op, 1, true);
4982 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4985 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4988 ctxt->memop.bytes = 1;
4989 if (ctxt->memop.type == OP_REG) {
4990 ctxt->memop.addr.reg = decode_register(ctxt,
4991 ctxt->modrm_rm, true);
4992 fetch_register_operand(&ctxt->memop);
4996 ctxt->memop.bytes = 2;
4999 ctxt->memop.bytes = 4;
5002 rc = decode_imm(ctxt, op, 2, false);
5005 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5009 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5011 register_address(ctxt, VCPU_REGS_RSI);
5012 op->addr.mem.seg = ctxt->seg_override;
5018 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5021 reg_read(ctxt, VCPU_REGS_RBX) +
5022 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5023 op->addr.mem.seg = ctxt->seg_override;
5028 op->addr.mem.ea = ctxt->_eip;
5029 op->bytes = ctxt->op_bytes + 2;
5030 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5033 ctxt->memop.bytes = ctxt->op_bytes + 2;
5037 op->val = VCPU_SREG_ES;
5041 op->val = VCPU_SREG_CS;
5045 op->val = VCPU_SREG_SS;
5049 op->val = VCPU_SREG_DS;
5053 op->val = VCPU_SREG_FS;
5057 op->val = VCPU_SREG_GS;
5060 /* Special instructions do their own operand decoding. */
5062 op->type = OP_NONE; /* Disable writeback. */
5070 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type)
5072 int rc = X86EMUL_CONTINUE;
5073 int mode = ctxt->mode;
5074 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5075 bool op_prefix = false;
5076 bool has_seg_override = false;
5077 struct opcode opcode;
5079 struct desc_struct desc;
5081 ctxt->memop.type = OP_NONE;
5082 ctxt->memopp = NULL;
5083 ctxt->_eip = ctxt->eip;
5084 ctxt->fetch.ptr = ctxt->fetch.data;
5085 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5086 ctxt->opcode_len = 1;
5087 ctxt->intercept = x86_intercept_none;
5089 memcpy(ctxt->fetch.data, insn, insn_len);
5091 rc = __do_insn_fetch_bytes(ctxt, 1);
5092 if (rc != X86EMUL_CONTINUE)
5097 case X86EMUL_MODE_REAL:
5098 case X86EMUL_MODE_VM86:
5099 def_op_bytes = def_ad_bytes = 2;
5100 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5102 def_op_bytes = def_ad_bytes = 4;
5104 case X86EMUL_MODE_PROT16:
5105 def_op_bytes = def_ad_bytes = 2;
5107 case X86EMUL_MODE_PROT32:
5108 def_op_bytes = def_ad_bytes = 4;
5110 #ifdef CONFIG_X86_64
5111 case X86EMUL_MODE_PROT64:
5117 return EMULATION_FAILED;
5120 ctxt->op_bytes = def_op_bytes;
5121 ctxt->ad_bytes = def_ad_bytes;
5123 /* Legacy prefixes. */
5125 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5126 case 0x66: /* operand-size override */
5128 /* switch between 2/4 bytes */
5129 ctxt->op_bytes = def_op_bytes ^ 6;
5131 case 0x67: /* address-size override */
5132 if (mode == X86EMUL_MODE_PROT64)
5133 /* switch between 4/8 bytes */
5134 ctxt->ad_bytes = def_ad_bytes ^ 12;
5136 /* switch between 2/4 bytes */
5137 ctxt->ad_bytes = def_ad_bytes ^ 6;
5139 case 0x26: /* ES override */
5140 has_seg_override = true;
5141 ctxt->seg_override = VCPU_SREG_ES;
5143 case 0x2e: /* CS override */
5144 has_seg_override = true;
5145 ctxt->seg_override = VCPU_SREG_CS;
5147 case 0x36: /* SS override */
5148 has_seg_override = true;
5149 ctxt->seg_override = VCPU_SREG_SS;
5151 case 0x3e: /* DS override */
5152 has_seg_override = true;
5153 ctxt->seg_override = VCPU_SREG_DS;
5155 case 0x64: /* FS override */
5156 has_seg_override = true;
5157 ctxt->seg_override = VCPU_SREG_FS;
5159 case 0x65: /* GS override */
5160 has_seg_override = true;
5161 ctxt->seg_override = VCPU_SREG_GS;
5163 case 0x40 ... 0x4f: /* REX */
5164 if (mode != X86EMUL_MODE_PROT64)
5166 ctxt->rex_prefix = ctxt->b;
5168 case 0xf0: /* LOCK */
5169 ctxt->lock_prefix = 1;
5171 case 0xf2: /* REPNE/REPNZ */
5172 case 0xf3: /* REP/REPE/REPZ */
5173 ctxt->rep_prefix = ctxt->b;
5179 /* Any legacy prefix after a REX prefix nullifies its effect. */
5181 ctxt->rex_prefix = 0;
5187 if (ctxt->rex_prefix & 8)
5188 ctxt->op_bytes = 8; /* REX.W */
5190 /* Opcode byte(s). */
5191 opcode = opcode_table[ctxt->b];
5192 /* Two-byte opcode? */
5193 if (ctxt->b == 0x0f) {
5194 ctxt->opcode_len = 2;
5195 ctxt->b = insn_fetch(u8, ctxt);
5196 opcode = twobyte_table[ctxt->b];
5198 /* 0F_38 opcode map */
5199 if (ctxt->b == 0x38) {
5200 ctxt->opcode_len = 3;
5201 ctxt->b = insn_fetch(u8, ctxt);
5202 opcode = opcode_map_0f_38[ctxt->b];
5205 ctxt->d = opcode.flags;
5207 if (ctxt->d & ModRM)
5208 ctxt->modrm = insn_fetch(u8, ctxt);
5210 /* vex-prefix instructions are not implemented */
5211 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5212 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5216 while (ctxt->d & GroupMask) {
5217 switch (ctxt->d & GroupMask) {
5219 goffset = (ctxt->modrm >> 3) & 7;
5220 opcode = opcode.u.group[goffset];
5223 goffset = (ctxt->modrm >> 3) & 7;
5224 if ((ctxt->modrm >> 6) == 3)
5225 opcode = opcode.u.gdual->mod3[goffset];
5227 opcode = opcode.u.gdual->mod012[goffset];
5230 goffset = ctxt->modrm & 7;
5231 opcode = opcode.u.group[goffset];
5234 if (ctxt->rep_prefix && op_prefix)
5235 return EMULATION_FAILED;
5236 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5237 switch (simd_prefix) {
5238 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5239 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5240 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5241 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5245 if (ctxt->modrm > 0xbf) {
5246 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5247 u32 index = array_index_nospec(
5248 ctxt->modrm - 0xc0, size);
5250 opcode = opcode.u.esc->high[index];
5252 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5256 if ((ctxt->modrm >> 6) == 3)
5257 opcode = opcode.u.idual->mod3;
5259 opcode = opcode.u.idual->mod012;
5262 if (ctxt->mode == X86EMUL_MODE_PROT64)
5263 opcode = opcode.u.mdual->mode64;
5265 opcode = opcode.u.mdual->mode32;
5268 return EMULATION_FAILED;
5271 ctxt->d &= ~(u64)GroupMask;
5272 ctxt->d |= opcode.flags;
5275 ctxt->is_branch = opcode.flags & IsBranch;
5279 return EMULATION_FAILED;
5281 ctxt->execute = opcode.u.execute;
5283 if (unlikely(emulation_type & EMULTYPE_TRAP_UD) &&
5284 likely(!(ctxt->d & EmulateOnUD)))
5285 return EMULATION_FAILED;
5287 if (unlikely(ctxt->d &
5288 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5291 * These are copied unconditionally here, and checked unconditionally
5292 * in x86_emulate_insn.
5294 ctxt->check_perm = opcode.check_perm;
5295 ctxt->intercept = opcode.intercept;
5297 if (ctxt->d & NotImpl)
5298 return EMULATION_FAILED;
5300 if (mode == X86EMUL_MODE_PROT64) {
5301 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5303 else if (ctxt->d & NearBranch)
5307 if (ctxt->d & Op3264) {
5308 if (mode == X86EMUL_MODE_PROT64)
5314 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5318 ctxt->op_bytes = 16;
5319 else if (ctxt->d & Mmx)
5323 /* ModRM and SIB bytes. */
5324 if (ctxt->d & ModRM) {
5325 rc = decode_modrm(ctxt, &ctxt->memop);
5326 if (!has_seg_override) {
5327 has_seg_override = true;
5328 ctxt->seg_override = ctxt->modrm_seg;
5330 } else if (ctxt->d & MemAbs)
5331 rc = decode_abs(ctxt, &ctxt->memop);
5332 if (rc != X86EMUL_CONTINUE)
5335 if (!has_seg_override)
5336 ctxt->seg_override = VCPU_SREG_DS;
5338 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5341 * Decode and fetch the source operand: register, memory
5344 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5345 if (rc != X86EMUL_CONTINUE)
5349 * Decode and fetch the second source operand: register, memory
5352 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5353 if (rc != X86EMUL_CONTINUE)
5356 /* Decode and fetch the destination operand: register or memory. */
5357 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5359 if (ctxt->rip_relative && likely(ctxt->memopp))
5360 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5361 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5364 if (rc == X86EMUL_PROPAGATE_FAULT)
5365 ctxt->have_exception = true;
5366 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5369 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5371 return ctxt->d & PageTable;
5374 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5376 /* The second termination condition only applies for REPE
5377 * and REPNE. Test if the repeat string operation prefix is
5378 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5379 * corresponding termination condition according to:
5380 * - if REPE/REPZ and ZF = 0 then done
5381 * - if REPNE/REPNZ and ZF = 1 then done
5383 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5384 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5385 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5386 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5387 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5388 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5394 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5399 rc = asm_safe("fwait");
5402 if (unlikely(rc != X86EMUL_CONTINUE))
5403 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5405 return X86EMUL_CONTINUE;
5408 static void fetch_possible_mmx_operand(struct operand *op)
5410 if (op->type == OP_MM)
5411 kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5414 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5416 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5418 if (!(ctxt->d & ByteOp))
5419 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5421 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5422 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5423 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5424 : "c"(ctxt->src2.val));
5426 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5427 if (!fop) /* exception is returned in fop variable */
5428 return emulate_de(ctxt);
5429 return X86EMUL_CONTINUE;
5432 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5434 /* Clear fields that are set conditionally but read without a guard. */
5435 ctxt->rip_relative = false;
5436 ctxt->rex_prefix = 0;
5437 ctxt->lock_prefix = 0;
5438 ctxt->rep_prefix = 0;
5439 ctxt->regs_valid = 0;
5440 ctxt->regs_dirty = 0;
5442 ctxt->io_read.pos = 0;
5443 ctxt->io_read.end = 0;
5444 ctxt->mem_read.end = 0;
5447 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5449 const struct x86_emulate_ops *ops = ctxt->ops;
5450 int rc = X86EMUL_CONTINUE;
5451 int saved_dst_type = ctxt->dst.type;
5452 unsigned emul_flags;
5454 ctxt->mem_read.pos = 0;
5456 /* LOCK prefix is allowed only with some instructions */
5457 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5458 rc = emulate_ud(ctxt);
5462 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5463 rc = emulate_ud(ctxt);
5467 emul_flags = ctxt->ops->get_hflags(ctxt);
5468 if (unlikely(ctxt->d &
5469 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5470 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5471 (ctxt->d & Undefined)) {
5472 rc = emulate_ud(ctxt);
5476 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5477 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5478 rc = emulate_ud(ctxt);
5482 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5483 rc = emulate_nm(ctxt);
5487 if (ctxt->d & Mmx) {
5488 rc = flush_pending_x87_faults(ctxt);
5489 if (rc != X86EMUL_CONTINUE)
5492 * Now that we know the fpu is exception safe, we can fetch
5495 fetch_possible_mmx_operand(&ctxt->src);
5496 fetch_possible_mmx_operand(&ctxt->src2);
5497 if (!(ctxt->d & Mov))
5498 fetch_possible_mmx_operand(&ctxt->dst);
5501 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5502 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5503 X86_ICPT_PRE_EXCEPT);
5504 if (rc != X86EMUL_CONTINUE)
5508 /* Instruction can only be executed in protected mode */
5509 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5510 rc = emulate_ud(ctxt);
5514 /* Privileged instruction can be executed only in CPL=0 */
5515 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5516 if (ctxt->d & PrivUD)
5517 rc = emulate_ud(ctxt);
5519 rc = emulate_gp(ctxt, 0);
5523 /* Do instruction specific permission checks */
5524 if (ctxt->d & CheckPerm) {
5525 rc = ctxt->check_perm(ctxt);
5526 if (rc != X86EMUL_CONTINUE)
5530 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5531 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5532 X86_ICPT_POST_EXCEPT);
5533 if (rc != X86EMUL_CONTINUE)
5537 if (ctxt->rep_prefix && (ctxt->d & String)) {
5538 /* All REP prefixes have the same first termination condition */
5539 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5540 string_registers_quirk(ctxt);
5541 ctxt->eip = ctxt->_eip;
5542 ctxt->eflags &= ~X86_EFLAGS_RF;
5548 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5549 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5550 ctxt->src.valptr, ctxt->src.bytes);
5551 if (rc != X86EMUL_CONTINUE)
5553 ctxt->src.orig_val64 = ctxt->src.val64;
5556 if (ctxt->src2.type == OP_MEM) {
5557 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5558 &ctxt->src2.val, ctxt->src2.bytes);
5559 if (rc != X86EMUL_CONTINUE)
5563 if ((ctxt->d & DstMask) == ImplicitOps)
5567 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5568 /* optimisation - avoid slow emulated read if Mov */
5569 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5570 &ctxt->dst.val, ctxt->dst.bytes);
5571 if (rc != X86EMUL_CONTINUE) {
5572 if (!(ctxt->d & NoWrite) &&
5573 rc == X86EMUL_PROPAGATE_FAULT &&
5574 ctxt->exception.vector == PF_VECTOR)
5575 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5579 /* Copy full 64-bit value for CMPXCHG8B. */
5580 ctxt->dst.orig_val64 = ctxt->dst.val64;
5584 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5585 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5586 X86_ICPT_POST_MEMACCESS);
5587 if (rc != X86EMUL_CONTINUE)
5591 if (ctxt->rep_prefix && (ctxt->d & String))
5592 ctxt->eflags |= X86_EFLAGS_RF;
5594 ctxt->eflags &= ~X86_EFLAGS_RF;
5596 if (ctxt->execute) {
5597 if (ctxt->d & Fastop)
5598 rc = fastop(ctxt, ctxt->fop);
5600 rc = ctxt->execute(ctxt);
5601 if (rc != X86EMUL_CONTINUE)
5606 if (ctxt->opcode_len == 2)
5608 else if (ctxt->opcode_len == 3)
5609 goto threebyte_insn;
5612 case 0x70 ... 0x7f: /* jcc (short) */
5613 if (test_cc(ctxt->b, ctxt->eflags))
5614 rc = jmp_rel(ctxt, ctxt->src.val);
5616 case 0x8d: /* lea r16/r32, m */
5617 ctxt->dst.val = ctxt->src.addr.mem.ea;
5619 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5620 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5621 ctxt->dst.type = OP_NONE;
5625 case 0x98: /* cbw/cwde/cdqe */
5626 switch (ctxt->op_bytes) {
5627 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5628 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5629 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5632 case 0xcc: /* int3 */
5633 rc = emulate_int(ctxt, 3);
5635 case 0xcd: /* int n */
5636 rc = emulate_int(ctxt, ctxt->src.val);
5638 case 0xce: /* into */
5639 if (ctxt->eflags & X86_EFLAGS_OF)
5640 rc = emulate_int(ctxt, 4);
5642 case 0xe9: /* jmp rel */
5643 case 0xeb: /* jmp rel short */
5644 rc = jmp_rel(ctxt, ctxt->src.val);
5645 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5647 case 0xf4: /* hlt */
5648 ctxt->ops->halt(ctxt);
5650 case 0xf5: /* cmc */
5651 /* complement carry flag from eflags reg */
5652 ctxt->eflags ^= X86_EFLAGS_CF;
5654 case 0xf8: /* clc */
5655 ctxt->eflags &= ~X86_EFLAGS_CF;
5657 case 0xf9: /* stc */
5658 ctxt->eflags |= X86_EFLAGS_CF;
5660 case 0xfc: /* cld */
5661 ctxt->eflags &= ~X86_EFLAGS_DF;
5663 case 0xfd: /* std */
5664 ctxt->eflags |= X86_EFLAGS_DF;
5667 goto cannot_emulate;
5670 if (rc != X86EMUL_CONTINUE)
5674 if (ctxt->d & SrcWrite) {
5675 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5676 rc = writeback(ctxt, &ctxt->src);
5677 if (rc != X86EMUL_CONTINUE)
5680 if (!(ctxt->d & NoWrite)) {
5681 rc = writeback(ctxt, &ctxt->dst);
5682 if (rc != X86EMUL_CONTINUE)
5687 * restore dst type in case the decoding will be reused
5688 * (happens for string instruction )
5690 ctxt->dst.type = saved_dst_type;
5692 if ((ctxt->d & SrcMask) == SrcSI)
5693 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5695 if ((ctxt->d & DstMask) == DstDI)
5696 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5698 if (ctxt->rep_prefix && (ctxt->d & String)) {
5700 struct read_cache *r = &ctxt->io_read;
5701 if ((ctxt->d & SrcMask) == SrcSI)
5702 count = ctxt->src.count;
5704 count = ctxt->dst.count;
5705 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5707 if (!string_insn_completed(ctxt)) {
5709 * Re-enter guest when pio read ahead buffer is empty
5710 * or, if it is not used, after each 1024 iteration.
5712 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5713 (r->end == 0 || r->end != r->pos)) {
5715 * Reset read cache. Usually happens before
5716 * decode, but since instruction is restarted
5717 * we have to do it here.
5719 ctxt->mem_read.end = 0;
5720 writeback_registers(ctxt);
5721 return EMULATION_RESTART;
5723 goto done; /* skip rip writeback */
5725 ctxt->eflags &= ~X86_EFLAGS_RF;
5728 ctxt->eip = ctxt->_eip;
5729 if (ctxt->mode != X86EMUL_MODE_PROT64)
5730 ctxt->eip = (u32)ctxt->_eip;
5733 if (rc == X86EMUL_PROPAGATE_FAULT) {
5734 if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt))
5735 return EMULATION_FAILED;
5736 ctxt->have_exception = true;
5738 if (rc == X86EMUL_INTERCEPTED)
5739 return EMULATION_INTERCEPTED;
5741 if (rc == X86EMUL_CONTINUE)
5742 writeback_registers(ctxt);
5744 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5748 case 0x09: /* wbinvd */
5749 (ctxt->ops->wbinvd)(ctxt);
5751 case 0x08: /* invd */
5752 case 0x0d: /* GrpP (prefetch) */
5753 case 0x18: /* Grp16 (prefetch/nop) */
5754 case 0x1f: /* nop */
5756 case 0x20: /* mov cr, reg */
5757 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5759 case 0x21: /* mov from dr to reg */
5760 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5762 case 0x40 ... 0x4f: /* cmov */
5763 if (test_cc(ctxt->b, ctxt->eflags))
5764 ctxt->dst.val = ctxt->src.val;
5765 else if (ctxt->op_bytes != 4)
5766 ctxt->dst.type = OP_NONE; /* no writeback */
5768 case 0x80 ... 0x8f: /* jnz rel, etc*/
5769 if (test_cc(ctxt->b, ctxt->eflags))
5770 rc = jmp_rel(ctxt, ctxt->src.val);
5772 case 0x90 ... 0x9f: /* setcc r/m8 */
5773 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5775 case 0xb6 ... 0xb7: /* movzx */
5776 ctxt->dst.bytes = ctxt->op_bytes;
5777 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5778 : (u16) ctxt->src.val;
5780 case 0xbe ... 0xbf: /* movsx */
5781 ctxt->dst.bytes = ctxt->op_bytes;
5782 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5783 (s16) ctxt->src.val;
5786 goto cannot_emulate;
5791 if (rc != X86EMUL_CONTINUE)
5797 return EMULATION_FAILED;
5800 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5802 invalidate_registers(ctxt);
5805 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5807 writeback_registers(ctxt);
5810 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5812 if (ctxt->rep_prefix && (ctxt->d & String))
5815 if (ctxt->d & TwoMemOp)