1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include "kvm_emulate.h"
24 #include <linux/stringify.h>
25 #include <asm/debugreg.h>
26 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
179 #define IsBranch ((u64)1 << 56) /* Instruction is considered a branch. */
181 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
183 #define X2(x...) x, x
184 #define X3(x...) X2(x), x
185 #define X4(x...) X2(x), X2(x)
186 #define X5(x...) X4(x), x
187 #define X6(x...) X4(x), X2(x)
188 #define X7(x...) X4(x), X3(x)
189 #define X8(x...) X4(x), X4(x)
190 #define X16(x...) X8(x), X8(x)
197 int (*execute)(struct x86_emulate_ctxt *ctxt);
198 const struct opcode *group;
199 const struct group_dual *gdual;
200 const struct gprefix *gprefix;
201 const struct escape *esc;
202 const struct instr_dual *idual;
203 const struct mode_dual *mdual;
204 void (*fastop)(struct fastop *fake);
206 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
210 struct opcode mod012[8];
211 struct opcode mod3[8];
215 struct opcode pfx_no;
216 struct opcode pfx_66;
217 struct opcode pfx_f2;
218 struct opcode pfx_f3;
223 struct opcode high[64];
227 struct opcode mod012;
232 struct opcode mode32;
233 struct opcode mode64;
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
238 enum x86_transfer_type {
240 X86_TRANSFER_CALL_JMP,
242 X86_TRANSFER_TASK_SWITCH,
245 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
247 if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))
248 nr &= NR_EMULATOR_GPRS - 1;
250 if (!(ctxt->regs_valid & (1 << nr))) {
251 ctxt->regs_valid |= 1 << nr;
252 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
254 return ctxt->_regs[nr];
257 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
259 if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))
260 nr &= NR_EMULATOR_GPRS - 1;
262 BUILD_BUG_ON(sizeof(ctxt->regs_dirty) * BITS_PER_BYTE < NR_EMULATOR_GPRS);
263 BUILD_BUG_ON(sizeof(ctxt->regs_valid) * BITS_PER_BYTE < NR_EMULATOR_GPRS);
265 ctxt->regs_valid |= 1 << nr;
266 ctxt->regs_dirty |= 1 << nr;
267 return &ctxt->_regs[nr];
270 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
273 return reg_write(ctxt, nr);
276 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
278 unsigned long dirty = ctxt->regs_dirty;
281 for_each_set_bit(reg, &dirty, NR_EMULATOR_GPRS)
282 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
285 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
287 ctxt->regs_dirty = 0;
288 ctxt->regs_valid = 0;
292 * These EFLAGS bits are restored from saved value during emulation, and
293 * any changes are written back to the saved value after emulation.
295 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
296 X86_EFLAGS_PF|X86_EFLAGS_CF)
305 * fastop functions have a special calling convention:
310 * flags: rflags (in/out)
311 * ex: rsi (in:fastop pointer, out:zero if exception)
313 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
314 * different operand sizes can be reached by calculation, rather than a jump
315 * table (which would be bigger than the code).
317 * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR
318 * and 1 for the straight line speculation INT3, leaves 7 bytes for the
319 * body of the function. Currently none is larger than 4.
321 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
323 #define FASTOP_SIZE 16
325 #define __FOP_FUNC(name) \
326 ".align " __stringify(FASTOP_SIZE) " \n\t" \
327 ".type " name ", @function \n\t" \
331 #define FOP_FUNC(name) \
334 #define __FOP_RET(name) \
336 ".size " name ", .-" name "\n\t"
338 #define FOP_RET(name) \
341 #define __FOP_START(op, align) \
342 extern void em_##op(struct fastop *fake); \
343 asm(".pushsection .text, \"ax\" \n\t" \
344 ".global em_" #op " \n\t" \
345 ".align " __stringify(align) " \n\t" \
348 #define FOP_START(op) __FOP_START(op, FASTOP_SIZE)
353 #define __FOPNOP(name) \
358 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
360 #define FOP1E(op, dst) \
361 __FOP_FUNC(#op "_" #dst) \
362 "10: " #op " %" #dst " \n\t" \
363 __FOP_RET(#op "_" #dst)
365 #define FOP1EEX(op, dst) \
366 FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
368 #define FASTOP1(op) \
373 ON64(FOP1E(op##q, rax)) \
376 /* 1-operand, using src2 (for MUL/DIV r/m) */
377 #define FASTOP1SRC2(op, name) \
382 ON64(FOP1E(op, rcx)) \
385 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
386 #define FASTOP1SRC2EX(op, name) \
391 ON64(FOP1EEX(op, rcx)) \
394 #define FOP2E(op, dst, src) \
395 __FOP_FUNC(#op "_" #dst "_" #src) \
396 #op " %" #src ", %" #dst " \n\t" \
397 __FOP_RET(#op "_" #dst "_" #src)
399 #define FASTOP2(op) \
401 FOP2E(op##b, al, dl) \
402 FOP2E(op##w, ax, dx) \
403 FOP2E(op##l, eax, edx) \
404 ON64(FOP2E(op##q, rax, rdx)) \
407 /* 2 operand, word only */
408 #define FASTOP2W(op) \
411 FOP2E(op##w, ax, dx) \
412 FOP2E(op##l, eax, edx) \
413 ON64(FOP2E(op##q, rax, rdx)) \
416 /* 2 operand, src is CL */
417 #define FASTOP2CL(op) \
419 FOP2E(op##b, al, cl) \
420 FOP2E(op##w, ax, cl) \
421 FOP2E(op##l, eax, cl) \
422 ON64(FOP2E(op##q, rax, cl)) \
425 /* 2 operand, src and dest are reversed */
426 #define FASTOP2R(op, name) \
428 FOP2E(op##b, dl, al) \
429 FOP2E(op##w, dx, ax) \
430 FOP2E(op##l, edx, eax) \
431 ON64(FOP2E(op##q, rdx, rax)) \
434 #define FOP3E(op, dst, src, src2) \
435 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
436 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
437 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
439 /* 3-operand, word-only, src2=cl */
440 #define FASTOP3WCL(op) \
443 FOP3E(op##w, ax, dx, cl) \
444 FOP3E(op##l, eax, edx, cl) \
445 ON64(FOP3E(op##q, rax, rdx, cl)) \
448 /* Special case for SETcc - 1 instruction per cc */
451 * Depending on .config the SETcc functions look like:
453 * ENDBR [4 bytes; CONFIG_X86_KERNEL_IBT]
454 * SETcc %al [3 bytes]
455 * RET | JMP __x86_return_thunk [1,5 bytes; CONFIG_RETHUNK]
456 * INT3 [1 byte; CONFIG_SLS]
458 #define SETCC_ALIGN 16
460 #define FOP_SETCC(op) \
461 ".align " __stringify(SETCC_ALIGN) " \n\t" \
462 ".type " #op ", @function \n\t" \
467 ".skip " __stringify(SETCC_ALIGN) " - (.-" #op "), 0xcc \n\t"
469 __FOP_START(setcc, SETCC_ALIGN)
490 "pushf; sbb %al, %al; popf \n\t"
495 * XXX: inoutclob user must know where the argument is being expanded.
496 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
498 #define asm_safe(insn, inoutclob...) \
502 asm volatile("1:" insn "\n" \
504 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[_fault]) \
505 : [_fault] "+r"(_fault) inoutclob ); \
507 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
510 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
511 enum x86_intercept intercept,
512 enum x86_intercept_stage stage)
514 struct x86_instruction_info info = {
515 .intercept = intercept,
516 .rep_prefix = ctxt->rep_prefix,
517 .modrm_mod = ctxt->modrm_mod,
518 .modrm_reg = ctxt->modrm_reg,
519 .modrm_rm = ctxt->modrm_rm,
520 .src_val = ctxt->src.val64,
521 .dst_val = ctxt->dst.val64,
522 .src_bytes = ctxt->src.bytes,
523 .dst_bytes = ctxt->dst.bytes,
524 .ad_bytes = ctxt->ad_bytes,
525 .next_rip = ctxt->eip,
528 return ctxt->ops->intercept(ctxt, &info, stage);
531 static void assign_masked(ulong *dest, ulong src, ulong mask)
533 *dest = (*dest & ~mask) | (src & mask);
536 static void assign_register(unsigned long *reg, u64 val, int bytes)
538 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
541 *(u8 *)reg = (u8)val;
544 *(u16 *)reg = (u16)val;
548 break; /* 64b: zero-extend */
555 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
557 return (1UL << (ctxt->ad_bytes << 3)) - 1;
560 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
563 struct desc_struct ss;
565 if (ctxt->mode == X86EMUL_MODE_PROT64)
567 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
568 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
571 static int stack_size(struct x86_emulate_ctxt *ctxt)
573 return (__fls(stack_mask(ctxt)) + 1) >> 3;
576 /* Access/update address held in a register, based on addressing mode. */
577 static inline unsigned long
578 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
580 if (ctxt->ad_bytes == sizeof(unsigned long))
583 return reg & ad_mask(ctxt);
586 static inline unsigned long
587 register_address(struct x86_emulate_ctxt *ctxt, int reg)
589 return address_mask(ctxt, reg_read(ctxt, reg));
592 static void masked_increment(ulong *reg, ulong mask, int inc)
594 assign_masked(reg, *reg + inc, mask);
598 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
600 ulong *preg = reg_rmw(ctxt, reg);
602 assign_register(preg, *preg + inc, ctxt->ad_bytes);
605 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
607 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
610 static u32 desc_limit_scaled(struct desc_struct *desc)
612 u32 limit = get_desc_limit(desc);
614 return desc->g ? (limit << 12) | 0xfff : limit;
617 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
619 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
622 return ctxt->ops->get_cached_segment_base(ctxt, seg);
625 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
626 u32 error, bool valid)
628 if (KVM_EMULATOR_BUG_ON(vec > 0x1f, ctxt))
629 return X86EMUL_UNHANDLEABLE;
631 ctxt->exception.vector = vec;
632 ctxt->exception.error_code = error;
633 ctxt->exception.error_code_valid = valid;
634 return X86EMUL_PROPAGATE_FAULT;
637 static int emulate_db(struct x86_emulate_ctxt *ctxt)
639 return emulate_exception(ctxt, DB_VECTOR, 0, false);
642 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
644 return emulate_exception(ctxt, GP_VECTOR, err, true);
647 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
649 return emulate_exception(ctxt, SS_VECTOR, err, true);
652 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
654 return emulate_exception(ctxt, UD_VECTOR, 0, false);
657 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
659 return emulate_exception(ctxt, TS_VECTOR, err, true);
662 static int emulate_de(struct x86_emulate_ctxt *ctxt)
664 return emulate_exception(ctxt, DE_VECTOR, 0, false);
667 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
669 return emulate_exception(ctxt, NM_VECTOR, 0, false);
672 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
675 struct desc_struct desc;
677 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
681 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
686 struct desc_struct desc;
688 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
689 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
692 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
694 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
697 static inline bool emul_is_noncanonical_address(u64 la,
698 struct x86_emulate_ctxt *ctxt)
700 return !__is_canonical_address(la, ctxt_virt_addr_bits(ctxt));
704 * x86 defines three classes of vector instructions: explicitly
705 * aligned, explicitly unaligned, and the rest, which change behaviour
706 * depending on whether they're AVX encoded or not.
708 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
709 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
710 * 512 bytes of data must be aligned to a 16 byte boundary.
712 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
714 u64 alignment = ctxt->d & AlignMask;
716 if (likely(size < 16))
731 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
732 struct segmented_address addr,
733 unsigned *max_size, unsigned size,
734 bool write, bool fetch,
735 enum x86emul_mode mode, ulong *linear)
737 struct desc_struct desc;
744 la = seg_base(ctxt, addr.seg) + addr.ea;
747 case X86EMUL_MODE_PROT64:
749 va_bits = ctxt_virt_addr_bits(ctxt);
750 if (!__is_canonical_address(la, va_bits))
753 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
754 if (size > *max_size)
758 *linear = la = (u32)la;
759 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
763 /* code segment in protected mode or read-only data segment */
764 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
765 || !(desc.type & 2)) && write)
767 /* unreadable code segment */
768 if (!fetch && (desc.type & 8) && !(desc.type & 2))
770 lim = desc_limit_scaled(&desc);
771 if (!(desc.type & 8) && (desc.type & 4)) {
772 /* expand-down segment */
775 lim = desc.d ? 0xffffffff : 0xffff;
779 if (lim == 0xffffffff)
782 *max_size = (u64)lim + 1 - addr.ea;
783 if (size > *max_size)
788 if (la & (insn_alignment(ctxt, size) - 1))
789 return emulate_gp(ctxt, 0);
790 return X86EMUL_CONTINUE;
792 if (addr.seg == VCPU_SREG_SS)
793 return emulate_ss(ctxt, 0);
795 return emulate_gp(ctxt, 0);
798 static int linearize(struct x86_emulate_ctxt *ctxt,
799 struct segmented_address addr,
800 unsigned size, bool write,
804 return __linearize(ctxt, addr, &max_size, size, write, false,
808 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
809 enum x86emul_mode mode)
814 struct segmented_address addr = { .seg = VCPU_SREG_CS,
817 if (ctxt->op_bytes != sizeof(unsigned long))
818 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
819 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
820 if (rc == X86EMUL_CONTINUE)
821 ctxt->_eip = addr.ea;
825 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
827 return assign_eip(ctxt, dst, ctxt->mode);
830 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
831 const struct desc_struct *cs_desc)
833 enum x86emul_mode mode = ctxt->mode;
837 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
841 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
843 mode = X86EMUL_MODE_PROT64;
845 mode = X86EMUL_MODE_PROT32; /* temporary value */
848 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
849 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
850 rc = assign_eip(ctxt, dst, mode);
851 if (rc == X86EMUL_CONTINUE)
856 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
858 return assign_eip_near(ctxt, ctxt->_eip + rel);
861 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
862 void *data, unsigned size)
864 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
867 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
868 ulong linear, void *data,
871 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
874 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
875 struct segmented_address addr,
882 rc = linearize(ctxt, addr, size, false, &linear);
883 if (rc != X86EMUL_CONTINUE)
885 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
888 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
889 struct segmented_address addr,
896 rc = linearize(ctxt, addr, size, true, &linear);
897 if (rc != X86EMUL_CONTINUE)
899 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
903 * Prefetch the remaining bytes of the instruction without crossing page
904 * boundary if they are not in fetch_cache yet.
906 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
909 unsigned size, max_size;
910 unsigned long linear;
911 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
912 struct segmented_address addr = { .seg = VCPU_SREG_CS,
913 .ea = ctxt->eip + cur_size };
916 * We do not know exactly how many bytes will be needed, and
917 * __linearize is expensive, so fetch as much as possible. We
918 * just have to avoid going beyond the 15 byte limit, the end
919 * of the segment, or the end of the page.
921 * __linearize is called with size 0 so that it does not do any
922 * boundary check itself. Instead, we use max_size to check
925 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
927 if (unlikely(rc != X86EMUL_CONTINUE))
930 size = min_t(unsigned, 15UL ^ cur_size, max_size);
931 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
934 * One instruction can only straddle two pages,
935 * and one has been loaded at the beginning of
936 * x86_decode_insn. So, if not enough bytes
937 * still, we must have hit the 15-byte boundary.
939 if (unlikely(size < op_size))
940 return emulate_gp(ctxt, 0);
942 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
943 size, &ctxt->exception);
944 if (unlikely(rc != X86EMUL_CONTINUE))
946 ctxt->fetch.end += size;
947 return X86EMUL_CONTINUE;
950 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
953 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
955 if (unlikely(done_size < size))
956 return __do_insn_fetch_bytes(ctxt, size - done_size);
958 return X86EMUL_CONTINUE;
961 /* Fetch next part of the instruction being emulated. */
962 #define insn_fetch(_type, _ctxt) \
965 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
966 if (rc != X86EMUL_CONTINUE) \
968 ctxt->_eip += sizeof(_type); \
969 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
970 ctxt->fetch.ptr += sizeof(_type); \
974 #define insn_fetch_arr(_arr, _size, _ctxt) \
976 rc = do_insn_fetch_bytes(_ctxt, _size); \
977 if (rc != X86EMUL_CONTINUE) \
979 ctxt->_eip += (_size); \
980 memcpy(_arr, ctxt->fetch.ptr, _size); \
981 ctxt->fetch.ptr += (_size); \
985 * Given the 'reg' portion of a ModRM byte, and a register block, return a
986 * pointer into the block that addresses the relevant register.
987 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
989 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
993 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
995 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
996 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
998 p = reg_rmw(ctxt, modrm_reg);
1002 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
1003 struct segmented_address addr,
1004 u16 *size, unsigned long *address, int op_bytes)
1011 rc = segmented_read_std(ctxt, addr, size, 2);
1012 if (rc != X86EMUL_CONTINUE)
1015 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1029 FASTOP1SRC2(mul, mul_ex);
1030 FASTOP1SRC2(imul, imul_ex);
1031 FASTOP1SRC2EX(div, div_ex);
1032 FASTOP1SRC2EX(idiv, idiv_ex);
1061 FASTOP2R(cmp, cmp_r);
1063 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1065 /* If src is zero, do not writeback, but update flags */
1066 if (ctxt->src.val == 0)
1067 ctxt->dst.type = OP_NONE;
1068 return fastop(ctxt, em_bsf);
1071 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1073 /* If src is zero, do not writeback, but update flags */
1074 if (ctxt->src.val == 0)
1075 ctxt->dst.type = OP_NONE;
1076 return fastop(ctxt, em_bsr);
1079 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1082 void (*fop)(void) = (void *)em_setcc + SETCC_ALIGN * (condition & 0xf);
1084 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1085 asm("push %[flags]; popf; " CALL_NOSPEC
1086 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1090 static void fetch_register_operand(struct operand *op)
1092 switch (op->bytes) {
1094 op->val = *(u8 *)op->addr.reg;
1097 op->val = *(u16 *)op->addr.reg;
1100 op->val = *(u32 *)op->addr.reg;
1103 op->val = *(u64 *)op->addr.reg;
1108 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1110 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1111 return emulate_nm(ctxt);
1114 asm volatile("fninit");
1116 return X86EMUL_CONTINUE;
1119 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1123 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1124 return emulate_nm(ctxt);
1127 asm volatile("fnstcw %0": "+m"(fcw));
1130 ctxt->dst.val = fcw;
1132 return X86EMUL_CONTINUE;
1135 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1139 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1140 return emulate_nm(ctxt);
1143 asm volatile("fnstsw %0": "+m"(fsw));
1146 ctxt->dst.val = fsw;
1148 return X86EMUL_CONTINUE;
1151 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1154 unsigned reg = ctxt->modrm_reg;
1156 if (!(ctxt->d & ModRM))
1157 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1159 if (ctxt->d & Sse) {
1163 kvm_read_sse_reg(reg, &op->vec_val);
1166 if (ctxt->d & Mmx) {
1175 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1176 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1178 fetch_register_operand(op);
1179 op->orig_val = op->val;
1182 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1184 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1185 ctxt->modrm_seg = VCPU_SREG_SS;
1188 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1192 int index_reg, base_reg, scale;
1193 int rc = X86EMUL_CONTINUE;
1196 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1197 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1198 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1200 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1201 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1202 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1203 ctxt->modrm_seg = VCPU_SREG_DS;
1205 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1207 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1208 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1210 if (ctxt->d & Sse) {
1213 op->addr.xmm = ctxt->modrm_rm;
1214 kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);
1217 if (ctxt->d & Mmx) {
1220 op->addr.mm = ctxt->modrm_rm & 7;
1223 fetch_register_operand(op);
1229 if (ctxt->ad_bytes == 2) {
1230 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1231 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1232 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1233 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1235 /* 16-bit ModR/M decode. */
1236 switch (ctxt->modrm_mod) {
1238 if (ctxt->modrm_rm == 6)
1239 modrm_ea += insn_fetch(u16, ctxt);
1242 modrm_ea += insn_fetch(s8, ctxt);
1245 modrm_ea += insn_fetch(u16, ctxt);
1248 switch (ctxt->modrm_rm) {
1250 modrm_ea += bx + si;
1253 modrm_ea += bx + di;
1256 modrm_ea += bp + si;
1259 modrm_ea += bp + di;
1268 if (ctxt->modrm_mod != 0)
1275 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1276 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1277 ctxt->modrm_seg = VCPU_SREG_SS;
1278 modrm_ea = (u16)modrm_ea;
1280 /* 32/64-bit ModR/M decode. */
1281 if ((ctxt->modrm_rm & 7) == 4) {
1282 sib = insn_fetch(u8, ctxt);
1283 index_reg |= (sib >> 3) & 7;
1284 base_reg |= sib & 7;
1287 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1288 modrm_ea += insn_fetch(s32, ctxt);
1290 modrm_ea += reg_read(ctxt, base_reg);
1291 adjust_modrm_seg(ctxt, base_reg);
1292 /* Increment ESP on POP [ESP] */
1293 if ((ctxt->d & IncSP) &&
1294 base_reg == VCPU_REGS_RSP)
1295 modrm_ea += ctxt->op_bytes;
1298 modrm_ea += reg_read(ctxt, index_reg) << scale;
1299 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1300 modrm_ea += insn_fetch(s32, ctxt);
1301 if (ctxt->mode == X86EMUL_MODE_PROT64)
1302 ctxt->rip_relative = 1;
1304 base_reg = ctxt->modrm_rm;
1305 modrm_ea += reg_read(ctxt, base_reg);
1306 adjust_modrm_seg(ctxt, base_reg);
1308 switch (ctxt->modrm_mod) {
1310 modrm_ea += insn_fetch(s8, ctxt);
1313 modrm_ea += insn_fetch(s32, ctxt);
1317 op->addr.mem.ea = modrm_ea;
1318 if (ctxt->ad_bytes != 8)
1319 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1325 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1328 int rc = X86EMUL_CONTINUE;
1331 switch (ctxt->ad_bytes) {
1333 op->addr.mem.ea = insn_fetch(u16, ctxt);
1336 op->addr.mem.ea = insn_fetch(u32, ctxt);
1339 op->addr.mem.ea = insn_fetch(u64, ctxt);
1346 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1350 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1351 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1353 if (ctxt->src.bytes == 2)
1354 sv = (s16)ctxt->src.val & (s16)mask;
1355 else if (ctxt->src.bytes == 4)
1356 sv = (s32)ctxt->src.val & (s32)mask;
1358 sv = (s64)ctxt->src.val & (s64)mask;
1360 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1361 ctxt->dst.addr.mem.ea + (sv >> 3));
1364 /* only subword offset */
1365 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1368 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1369 unsigned long addr, void *dest, unsigned size)
1372 struct read_cache *mc = &ctxt->mem_read;
1374 if (mc->pos < mc->end)
1377 if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))
1378 return X86EMUL_UNHANDLEABLE;
1380 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1382 if (rc != X86EMUL_CONTINUE)
1388 memcpy(dest, mc->data + mc->pos, size);
1390 return X86EMUL_CONTINUE;
1393 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1394 struct segmented_address addr,
1401 rc = linearize(ctxt, addr, size, false, &linear);
1402 if (rc != X86EMUL_CONTINUE)
1404 return read_emulated(ctxt, linear, data, size);
1407 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1408 struct segmented_address addr,
1415 rc = linearize(ctxt, addr, size, true, &linear);
1416 if (rc != X86EMUL_CONTINUE)
1418 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1422 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1423 struct segmented_address addr,
1424 const void *orig_data, const void *data,
1430 rc = linearize(ctxt, addr, size, true, &linear);
1431 if (rc != X86EMUL_CONTINUE)
1433 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1434 size, &ctxt->exception);
1437 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1438 unsigned int size, unsigned short port,
1441 struct read_cache *rc = &ctxt->io_read;
1443 if (rc->pos == rc->end) { /* refill pio read ahead */
1444 unsigned int in_page, n;
1445 unsigned int count = ctxt->rep_prefix ?
1446 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1447 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1448 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1449 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1450 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1453 rc->pos = rc->end = 0;
1454 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1459 if (ctxt->rep_prefix && (ctxt->d & String) &&
1460 !(ctxt->eflags & X86_EFLAGS_DF)) {
1461 ctxt->dst.data = rc->data + rc->pos;
1462 ctxt->dst.type = OP_MEM_STR;
1463 ctxt->dst.count = (rc->end - rc->pos) / size;
1466 memcpy(dest, rc->data + rc->pos, size);
1472 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1473 u16 index, struct desc_struct *desc)
1478 ctxt->ops->get_idt(ctxt, &dt);
1480 if (dt.size < index * 8 + 7)
1481 return emulate_gp(ctxt, index << 3 | 0x2);
1483 addr = dt.address + index * 8;
1484 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1487 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1488 u16 selector, struct desc_ptr *dt)
1490 const struct x86_emulate_ops *ops = ctxt->ops;
1493 if (selector & 1 << 2) {
1494 struct desc_struct desc;
1497 memset(dt, 0, sizeof(*dt));
1498 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1502 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1503 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1505 ops->get_gdt(ctxt, dt);
1508 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1509 u16 selector, ulong *desc_addr_p)
1512 u16 index = selector >> 3;
1515 get_descriptor_table_ptr(ctxt, selector, &dt);
1517 if (dt.size < index * 8 + 7)
1518 return emulate_gp(ctxt, selector & 0xfffc);
1520 addr = dt.address + index * 8;
1522 #ifdef CONFIG_X86_64
1523 if (addr >> 32 != 0) {
1526 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1527 if (!(efer & EFER_LMA))
1532 *desc_addr_p = addr;
1533 return X86EMUL_CONTINUE;
1536 /* allowed just for 8 bytes segments */
1537 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1538 u16 selector, struct desc_struct *desc,
1543 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1544 if (rc != X86EMUL_CONTINUE)
1547 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1550 /* allowed just for 8 bytes segments */
1551 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1552 u16 selector, struct desc_struct *desc)
1557 rc = get_descriptor_ptr(ctxt, selector, &addr);
1558 if (rc != X86EMUL_CONTINUE)
1561 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1564 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1565 u16 selector, int seg, u8 cpl,
1566 enum x86_transfer_type transfer,
1567 struct desc_struct *desc)
1569 struct desc_struct seg_desc, old_desc;
1571 unsigned err_vec = GP_VECTOR;
1573 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1579 memset(&seg_desc, 0, sizeof(seg_desc));
1581 if (ctxt->mode == X86EMUL_MODE_REAL) {
1582 /* set real mode segment descriptor (keep limit etc. for
1584 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1585 set_desc_base(&seg_desc, selector << 4);
1587 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1588 /* VM86 needs a clean new segment descriptor */
1589 set_desc_base(&seg_desc, selector << 4);
1590 set_desc_limit(&seg_desc, 0xffff);
1600 /* TR should be in GDT only */
1601 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1604 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1605 if (null_selector) {
1606 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1609 if (seg == VCPU_SREG_SS) {
1610 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1614 * ctxt->ops->set_segment expects the CPL to be in
1615 * SS.DPL, so fake an expand-up 32-bit data segment.
1625 /* Skip all following checks */
1629 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1630 if (ret != X86EMUL_CONTINUE)
1633 err_code = selector & 0xfffc;
1634 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1637 /* can't load system descriptor into segment selector */
1638 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1639 if (transfer == X86_TRANSFER_CALL_JMP)
1640 return X86EMUL_UNHANDLEABLE;
1649 * segment is not a writable data segment or segment
1650 * selector's RPL != CPL or segment selector's RPL != CPL
1652 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1656 if (!(seg_desc.type & 8))
1659 if (transfer == X86_TRANSFER_RET) {
1660 /* RET can never return to an inner privilege level. */
1663 /* Outer-privilege level return is not implemented */
1665 return X86EMUL_UNHANDLEABLE;
1667 if (transfer == X86_TRANSFER_RET || transfer == X86_TRANSFER_TASK_SWITCH) {
1668 if (seg_desc.type & 4) {
1677 } else { /* X86_TRANSFER_CALL_JMP */
1678 if (seg_desc.type & 4) {
1684 if (rpl > cpl || dpl != cpl)
1688 /* in long-mode d/b must be clear if l is set */
1689 if (seg_desc.d && seg_desc.l) {
1692 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1693 if (efer & EFER_LMA)
1697 /* CS(RPL) <- CPL */
1698 selector = (selector & 0xfffc) | cpl;
1701 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1704 case VCPU_SREG_LDTR:
1705 if (seg_desc.s || seg_desc.type != 2)
1708 default: /* DS, ES, FS, or GS */
1710 * segment is not a data or readable code segment or
1711 * ((segment is a data or nonconforming code segment)
1712 * and (both RPL and CPL > DPL))
1714 if ((seg_desc.type & 0xa) == 0x8 ||
1715 (((seg_desc.type & 0xc) != 0xc) &&
1716 (rpl > dpl && cpl > dpl)))
1722 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1727 /* mark segment as accessed */
1728 if (!(seg_desc.type & 1)) {
1730 ret = write_segment_descriptor(ctxt, selector,
1732 if (ret != X86EMUL_CONTINUE)
1735 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1736 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1737 if (ret != X86EMUL_CONTINUE)
1739 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1740 ((u64)base3 << 32), ctxt))
1741 return emulate_gp(ctxt, err_code);
1744 if (seg == VCPU_SREG_TR) {
1745 old_desc = seg_desc;
1746 seg_desc.type |= 2; /* busy */
1747 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1748 sizeof(seg_desc), &ctxt->exception);
1749 if (ret != X86EMUL_CONTINUE)
1753 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1756 return X86EMUL_CONTINUE;
1758 return emulate_exception(ctxt, err_vec, err_code, true);
1761 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1762 u16 selector, int seg)
1764 u8 cpl = ctxt->ops->cpl(ctxt);
1767 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1768 * they can load it at CPL<3 (Intel's manual says only LSS can,
1771 * However, the Intel manual says that putting IST=1/DPL=3 in
1772 * an interrupt gate will result in SS=3 (the AMD manual instead
1773 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1774 * and only forbid it here.
1776 if (seg == VCPU_SREG_SS && selector == 3 &&
1777 ctxt->mode == X86EMUL_MODE_PROT64)
1778 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1780 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1781 X86_TRANSFER_NONE, NULL);
1784 static void write_register_operand(struct operand *op)
1786 return assign_register(op->addr.reg, op->val, op->bytes);
1789 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1793 write_register_operand(op);
1796 if (ctxt->lock_prefix)
1797 return segmented_cmpxchg(ctxt,
1803 return segmented_write(ctxt,
1809 return segmented_write(ctxt,
1812 op->bytes * op->count);
1815 kvm_write_sse_reg(op->addr.xmm, &op->vec_val);
1818 kvm_write_mmx_reg(op->addr.mm, &op->mm_val);
1826 return X86EMUL_CONTINUE;
1829 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1831 struct segmented_address addr;
1833 rsp_increment(ctxt, -bytes);
1834 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1835 addr.seg = VCPU_SREG_SS;
1837 return segmented_write(ctxt, addr, data, bytes);
1840 static int em_push(struct x86_emulate_ctxt *ctxt)
1842 /* Disable writeback. */
1843 ctxt->dst.type = OP_NONE;
1844 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1847 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1848 void *dest, int len)
1851 struct segmented_address addr;
1853 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1854 addr.seg = VCPU_SREG_SS;
1855 rc = segmented_read(ctxt, addr, dest, len);
1856 if (rc != X86EMUL_CONTINUE)
1859 rsp_increment(ctxt, len);
1863 static int em_pop(struct x86_emulate_ctxt *ctxt)
1865 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1868 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1869 void *dest, int len)
1872 unsigned long val, change_mask;
1873 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1874 int cpl = ctxt->ops->cpl(ctxt);
1876 rc = emulate_pop(ctxt, &val, len);
1877 if (rc != X86EMUL_CONTINUE)
1880 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1881 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1882 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1883 X86_EFLAGS_AC | X86_EFLAGS_ID;
1885 switch(ctxt->mode) {
1886 case X86EMUL_MODE_PROT64:
1887 case X86EMUL_MODE_PROT32:
1888 case X86EMUL_MODE_PROT16:
1890 change_mask |= X86_EFLAGS_IOPL;
1892 change_mask |= X86_EFLAGS_IF;
1894 case X86EMUL_MODE_VM86:
1896 return emulate_gp(ctxt, 0);
1897 change_mask |= X86_EFLAGS_IF;
1899 default: /* real mode */
1900 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1904 *(unsigned long *)dest =
1905 (ctxt->eflags & ~change_mask) | (val & change_mask);
1910 static int em_popf(struct x86_emulate_ctxt *ctxt)
1912 ctxt->dst.type = OP_REG;
1913 ctxt->dst.addr.reg = &ctxt->eflags;
1914 ctxt->dst.bytes = ctxt->op_bytes;
1915 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1918 static int em_enter(struct x86_emulate_ctxt *ctxt)
1921 unsigned frame_size = ctxt->src.val;
1922 unsigned nesting_level = ctxt->src2.val & 31;
1926 return X86EMUL_UNHANDLEABLE;
1928 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1929 rc = push(ctxt, &rbp, stack_size(ctxt));
1930 if (rc != X86EMUL_CONTINUE)
1932 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1934 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1935 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1937 return X86EMUL_CONTINUE;
1940 static int em_leave(struct x86_emulate_ctxt *ctxt)
1942 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1944 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1947 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1949 int seg = ctxt->src2.val;
1951 ctxt->src.val = get_segment_selector(ctxt, seg);
1952 if (ctxt->op_bytes == 4) {
1953 rsp_increment(ctxt, -2);
1957 return em_push(ctxt);
1960 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1962 int seg = ctxt->src2.val;
1963 unsigned long selector;
1966 rc = emulate_pop(ctxt, &selector, 2);
1967 if (rc != X86EMUL_CONTINUE)
1970 if (ctxt->modrm_reg == VCPU_SREG_SS)
1971 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1972 if (ctxt->op_bytes > 2)
1973 rsp_increment(ctxt, ctxt->op_bytes - 2);
1975 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1979 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1981 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1982 int rc = X86EMUL_CONTINUE;
1983 int reg = VCPU_REGS_RAX;
1985 while (reg <= VCPU_REGS_RDI) {
1986 (reg == VCPU_REGS_RSP) ?
1987 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1990 if (rc != X86EMUL_CONTINUE)
1999 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2001 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2002 return em_push(ctxt);
2005 static int em_popa(struct x86_emulate_ctxt *ctxt)
2007 int rc = X86EMUL_CONTINUE;
2008 int reg = VCPU_REGS_RDI;
2011 while (reg >= VCPU_REGS_RAX) {
2012 if (reg == VCPU_REGS_RSP) {
2013 rsp_increment(ctxt, ctxt->op_bytes);
2017 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2018 if (rc != X86EMUL_CONTINUE)
2020 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2026 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2028 const struct x86_emulate_ops *ops = ctxt->ops;
2035 /* TODO: Add limit checks */
2036 ctxt->src.val = ctxt->eflags;
2038 if (rc != X86EMUL_CONTINUE)
2041 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2043 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2045 if (rc != X86EMUL_CONTINUE)
2048 ctxt->src.val = ctxt->_eip;
2050 if (rc != X86EMUL_CONTINUE)
2053 ops->get_idt(ctxt, &dt);
2055 eip_addr = dt.address + (irq << 2);
2056 cs_addr = dt.address + (irq << 2) + 2;
2058 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2059 if (rc != X86EMUL_CONTINUE)
2062 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2063 if (rc != X86EMUL_CONTINUE)
2066 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2067 if (rc != X86EMUL_CONTINUE)
2075 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2079 invalidate_registers(ctxt);
2080 rc = __emulate_int_real(ctxt, irq);
2081 if (rc == X86EMUL_CONTINUE)
2082 writeback_registers(ctxt);
2086 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2088 switch(ctxt->mode) {
2089 case X86EMUL_MODE_REAL:
2090 return __emulate_int_real(ctxt, irq);
2091 case X86EMUL_MODE_VM86:
2092 case X86EMUL_MODE_PROT16:
2093 case X86EMUL_MODE_PROT32:
2094 case X86EMUL_MODE_PROT64:
2096 /* Protected mode interrupts unimplemented yet */
2097 return X86EMUL_UNHANDLEABLE;
2101 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2103 int rc = X86EMUL_CONTINUE;
2104 unsigned long temp_eip = 0;
2105 unsigned long temp_eflags = 0;
2106 unsigned long cs = 0;
2107 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2108 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2109 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2110 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2111 X86_EFLAGS_AC | X86_EFLAGS_ID |
2113 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2116 /* TODO: Add stack limit check */
2118 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2120 if (rc != X86EMUL_CONTINUE)
2123 if (temp_eip & ~0xffff)
2124 return emulate_gp(ctxt, 0);
2126 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2128 if (rc != X86EMUL_CONTINUE)
2131 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2133 if (rc != X86EMUL_CONTINUE)
2136 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2138 if (rc != X86EMUL_CONTINUE)
2141 ctxt->_eip = temp_eip;
2143 if (ctxt->op_bytes == 4)
2144 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2145 else if (ctxt->op_bytes == 2) {
2146 ctxt->eflags &= ~0xffff;
2147 ctxt->eflags |= temp_eflags;
2150 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2151 ctxt->eflags |= X86_EFLAGS_FIXED;
2152 ctxt->ops->set_nmi_mask(ctxt, false);
2157 static int em_iret(struct x86_emulate_ctxt *ctxt)
2159 switch(ctxt->mode) {
2160 case X86EMUL_MODE_REAL:
2161 return emulate_iret_real(ctxt);
2162 case X86EMUL_MODE_VM86:
2163 case X86EMUL_MODE_PROT16:
2164 case X86EMUL_MODE_PROT32:
2165 case X86EMUL_MODE_PROT64:
2167 /* iret from protected mode unimplemented yet */
2168 return X86EMUL_UNHANDLEABLE;
2172 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2176 struct desc_struct new_desc;
2177 u8 cpl = ctxt->ops->cpl(ctxt);
2179 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2181 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2182 X86_TRANSFER_CALL_JMP,
2184 if (rc != X86EMUL_CONTINUE)
2187 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2188 /* Error handling is not implemented. */
2189 if (rc != X86EMUL_CONTINUE)
2190 return X86EMUL_UNHANDLEABLE;
2195 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2197 return assign_eip_near(ctxt, ctxt->src.val);
2200 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2205 old_eip = ctxt->_eip;
2206 rc = assign_eip_near(ctxt, ctxt->src.val);
2207 if (rc != X86EMUL_CONTINUE)
2209 ctxt->src.val = old_eip;
2214 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2216 u64 old = ctxt->dst.orig_val64;
2218 if (ctxt->dst.bytes == 16)
2219 return X86EMUL_UNHANDLEABLE;
2221 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2222 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2223 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2224 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2225 ctxt->eflags &= ~X86_EFLAGS_ZF;
2227 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2228 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2230 ctxt->eflags |= X86_EFLAGS_ZF;
2232 return X86EMUL_CONTINUE;
2235 static int em_ret(struct x86_emulate_ctxt *ctxt)
2240 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2241 if (rc != X86EMUL_CONTINUE)
2244 return assign_eip_near(ctxt, eip);
2247 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2250 unsigned long eip, cs;
2251 int cpl = ctxt->ops->cpl(ctxt);
2252 struct desc_struct new_desc;
2254 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2255 if (rc != X86EMUL_CONTINUE)
2257 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2258 if (rc != X86EMUL_CONTINUE)
2260 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2263 if (rc != X86EMUL_CONTINUE)
2265 rc = assign_eip_far(ctxt, eip, &new_desc);
2266 /* Error handling is not implemented. */
2267 if (rc != X86EMUL_CONTINUE)
2268 return X86EMUL_UNHANDLEABLE;
2273 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2277 rc = em_ret_far(ctxt);
2278 if (rc != X86EMUL_CONTINUE)
2280 rsp_increment(ctxt, ctxt->src.val);
2281 return X86EMUL_CONTINUE;
2284 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2286 /* Save real source value, then compare EAX against destination. */
2287 ctxt->dst.orig_val = ctxt->dst.val;
2288 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2289 ctxt->src.orig_val = ctxt->src.val;
2290 ctxt->src.val = ctxt->dst.orig_val;
2291 fastop(ctxt, em_cmp);
2293 if (ctxt->eflags & X86_EFLAGS_ZF) {
2294 /* Success: write back to memory; no update of EAX */
2295 ctxt->src.type = OP_NONE;
2296 ctxt->dst.val = ctxt->src.orig_val;
2298 /* Failure: write the value we saw to EAX. */
2299 ctxt->src.type = OP_REG;
2300 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2301 ctxt->src.val = ctxt->dst.orig_val;
2302 /* Create write-cycle to dest by writing the same value */
2303 ctxt->dst.val = ctxt->dst.orig_val;
2305 return X86EMUL_CONTINUE;
2308 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2310 int seg = ctxt->src2.val;
2314 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2316 rc = load_segment_descriptor(ctxt, sel, seg);
2317 if (rc != X86EMUL_CONTINUE)
2320 ctxt->dst.val = ctxt->src.val;
2324 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2326 #ifdef CONFIG_X86_64
2327 return ctxt->ops->guest_has_long_mode(ctxt);
2333 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2335 desc->g = (flags >> 23) & 1;
2336 desc->d = (flags >> 22) & 1;
2337 desc->l = (flags >> 21) & 1;
2338 desc->avl = (flags >> 20) & 1;
2339 desc->p = (flags >> 15) & 1;
2340 desc->dpl = (flags >> 13) & 3;
2341 desc->s = (flags >> 12) & 1;
2342 desc->type = (flags >> 8) & 15;
2345 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2348 struct desc_struct desc;
2352 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2355 offset = 0x7f84 + n * 12;
2357 offset = 0x7f2c + (n - 3) * 12;
2359 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2360 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2361 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2362 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2363 return X86EMUL_CONTINUE;
2366 #ifdef CONFIG_X86_64
2367 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2370 struct desc_struct desc;
2375 offset = 0x7e00 + n * 16;
2377 selector = GET_SMSTATE(u16, smstate, offset);
2378 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2379 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2380 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2381 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2383 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2384 return X86EMUL_CONTINUE;
2388 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2389 u64 cr0, u64 cr3, u64 cr4)
2394 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2396 if (cr4 & X86_CR4_PCIDE) {
2401 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2403 return X86EMUL_UNHANDLEABLE;
2406 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2407 * Then enable protected mode. However, PCID cannot be enabled
2408 * if EFER.LMA=0, so set it separately.
2410 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2412 return X86EMUL_UNHANDLEABLE;
2414 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2416 return X86EMUL_UNHANDLEABLE;
2418 if (cr4 & X86_CR4_PCIDE) {
2419 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2421 return X86EMUL_UNHANDLEABLE;
2423 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2425 return X86EMUL_UNHANDLEABLE;
2430 return X86EMUL_CONTINUE;
2433 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2434 const char *smstate)
2436 struct desc_struct desc;
2439 u32 val, cr0, cr3, cr4;
2442 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2443 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2444 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2445 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2447 for (i = 0; i < NR_EMULATOR_GPRS; i++)
2448 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2450 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2452 if (ctxt->ops->set_dr(ctxt, 6, val))
2453 return X86EMUL_UNHANDLEABLE;
2455 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2457 if (ctxt->ops->set_dr(ctxt, 7, val))
2458 return X86EMUL_UNHANDLEABLE;
2460 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2461 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2462 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2463 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2464 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2466 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2467 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2468 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2469 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2470 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2472 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2473 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2474 ctxt->ops->set_gdt(ctxt, &dt);
2476 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2477 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2478 ctxt->ops->set_idt(ctxt, &dt);
2480 for (i = 0; i < 6; i++) {
2481 int r = rsm_load_seg_32(ctxt, smstate, i);
2482 if (r != X86EMUL_CONTINUE)
2486 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2488 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2490 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2493 #ifdef CONFIG_X86_64
2494 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2495 const char *smstate)
2497 struct desc_struct desc;
2499 u64 val, cr0, cr3, cr4;
2504 for (i = 0; i < NR_EMULATOR_GPRS; i++)
2505 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2507 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2508 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2510 val = GET_SMSTATE(u64, smstate, 0x7f68);
2512 if (ctxt->ops->set_dr(ctxt, 6, val))
2513 return X86EMUL_UNHANDLEABLE;
2515 val = GET_SMSTATE(u64, smstate, 0x7f60);
2517 if (ctxt->ops->set_dr(ctxt, 7, val))
2518 return X86EMUL_UNHANDLEABLE;
2520 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2521 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2522 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2523 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2524 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2526 if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))
2527 return X86EMUL_UNHANDLEABLE;
2529 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2530 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2531 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2532 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2533 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2534 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2536 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2537 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2538 ctxt->ops->set_idt(ctxt, &dt);
2540 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2541 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2542 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2543 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2544 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2545 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2547 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2548 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2549 ctxt->ops->set_gdt(ctxt, &dt);
2551 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2552 if (r != X86EMUL_CONTINUE)
2555 for (i = 0; i < 6; i++) {
2556 r = rsm_load_seg_64(ctxt, smstate, i);
2557 if (r != X86EMUL_CONTINUE)
2561 return X86EMUL_CONTINUE;
2565 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2567 unsigned long cr0, cr4, efer;
2572 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2573 return emulate_ud(ctxt);
2575 smbase = ctxt->ops->get_smbase(ctxt);
2577 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2578 if (ret != X86EMUL_CONTINUE)
2579 return X86EMUL_UNHANDLEABLE;
2581 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2582 ctxt->ops->set_nmi_mask(ctxt, false);
2584 ctxt->ops->exiting_smm(ctxt);
2587 * Get back to real mode, to prepare a safe state in which to load
2588 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2589 * supports long mode.
2591 if (emulator_has_longmode(ctxt)) {
2592 struct desc_struct cs_desc;
2594 /* Zero CR4.PCIDE before CR0.PG. */
2595 cr4 = ctxt->ops->get_cr(ctxt, 4);
2596 if (cr4 & X86_CR4_PCIDE)
2597 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2599 /* A 32-bit code segment is required to clear EFER.LMA. */
2600 memset(&cs_desc, 0, sizeof(cs_desc));
2602 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2603 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2606 /* For the 64-bit case, this will clear EFER.LMA. */
2607 cr0 = ctxt->ops->get_cr(ctxt, 0);
2608 if (cr0 & X86_CR0_PE)
2609 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2611 if (emulator_has_longmode(ctxt)) {
2612 /* Clear CR4.PAE before clearing EFER.LME. */
2613 cr4 = ctxt->ops->get_cr(ctxt, 4);
2614 if (cr4 & X86_CR4_PAE)
2615 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2617 /* And finally go back to 32-bit mode. */
2619 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2623 * Give leave_smm() a chance to make ISA-specific changes to the vCPU
2624 * state (e.g. enter guest mode) before loading state from the SMM
2627 if (ctxt->ops->leave_smm(ctxt, buf))
2628 goto emulate_shutdown;
2630 #ifdef CONFIG_X86_64
2631 if (emulator_has_longmode(ctxt))
2632 ret = rsm_load_state_64(ctxt, buf);
2635 ret = rsm_load_state_32(ctxt, buf);
2637 if (ret != X86EMUL_CONTINUE)
2638 goto emulate_shutdown;
2641 * Note, the ctxt->ops callbacks are responsible for handling side
2642 * effects when writing MSRs and CRs, e.g. MMU context resets, CPUID
2643 * runtime updates, etc... If that changes, e.g. this flow is moved
2644 * out of the emulator to make it look more like enter_smm(), then
2645 * those side effects need to be explicitly handled for both success
2648 return X86EMUL_CONTINUE;
2651 ctxt->ops->triple_fault(ctxt);
2652 return X86EMUL_CONTINUE;
2656 setup_syscalls_segments(struct desc_struct *cs, struct desc_struct *ss)
2658 cs->l = 0; /* will be adjusted later */
2659 set_desc_base(cs, 0); /* flat segment */
2660 cs->g = 1; /* 4kb granularity */
2661 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2662 cs->type = 0x0b; /* Read, Execute, Accessed */
2664 cs->dpl = 0; /* will be adjusted later */
2669 set_desc_base(ss, 0); /* flat segment */
2670 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2671 ss->g = 1; /* 4kb granularity */
2673 ss->type = 0x03; /* Read/Write, Accessed */
2674 ss->d = 1; /* 32bit stack segment */
2681 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2683 u32 eax, ebx, ecx, edx;
2686 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2687 return is_guest_vendor_intel(ebx, ecx, edx);
2690 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2692 const struct x86_emulate_ops *ops = ctxt->ops;
2693 u32 eax, ebx, ecx, edx;
2696 * syscall should always be enabled in longmode - so only become
2697 * vendor specific (cpuid) if other modes are active...
2699 if (ctxt->mode == X86EMUL_MODE_PROT64)
2704 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2706 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
2707 * 64bit guest with a 32bit compat-app running will #UD !! While this
2708 * behaviour can be fixed (by emulating) into AMD response - CPUs of
2709 * AMD can't behave like Intel.
2711 if (is_guest_vendor_intel(ebx, ecx, edx))
2714 if (is_guest_vendor_amd(ebx, ecx, edx) ||
2715 is_guest_vendor_hygon(ebx, ecx, edx))
2719 * default: (not Intel, not AMD, not Hygon), apply Intel's
2725 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2727 const struct x86_emulate_ops *ops = ctxt->ops;
2728 struct desc_struct cs, ss;
2733 /* syscall is not available in real mode */
2734 if (ctxt->mode == X86EMUL_MODE_REAL ||
2735 ctxt->mode == X86EMUL_MODE_VM86)
2736 return emulate_ud(ctxt);
2738 if (!(em_syscall_is_enabled(ctxt)))
2739 return emulate_ud(ctxt);
2741 ops->get_msr(ctxt, MSR_EFER, &efer);
2742 if (!(efer & EFER_SCE))
2743 return emulate_ud(ctxt);
2745 setup_syscalls_segments(&cs, &ss);
2746 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2748 cs_sel = (u16)(msr_data & 0xfffc);
2749 ss_sel = (u16)(msr_data + 8);
2751 if (efer & EFER_LMA) {
2755 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2756 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2758 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2759 if (efer & EFER_LMA) {
2760 #ifdef CONFIG_X86_64
2761 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2764 ctxt->mode == X86EMUL_MODE_PROT64 ?
2765 MSR_LSTAR : MSR_CSTAR, &msr_data);
2766 ctxt->_eip = msr_data;
2768 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2769 ctxt->eflags &= ~msr_data;
2770 ctxt->eflags |= X86_EFLAGS_FIXED;
2774 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2775 ctxt->_eip = (u32)msr_data;
2777 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2780 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2781 return X86EMUL_CONTINUE;
2784 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2786 const struct x86_emulate_ops *ops = ctxt->ops;
2787 struct desc_struct cs, ss;
2792 ops->get_msr(ctxt, MSR_EFER, &efer);
2793 /* inject #GP if in real mode */
2794 if (ctxt->mode == X86EMUL_MODE_REAL)
2795 return emulate_gp(ctxt, 0);
2798 * Not recognized on AMD in compat mode (but is recognized in legacy
2801 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2802 && !vendor_intel(ctxt))
2803 return emulate_ud(ctxt);
2805 /* sysenter/sysexit have not been tested in 64bit mode. */
2806 if (ctxt->mode == X86EMUL_MODE_PROT64)
2807 return X86EMUL_UNHANDLEABLE;
2809 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2810 if ((msr_data & 0xfffc) == 0x0)
2811 return emulate_gp(ctxt, 0);
2813 setup_syscalls_segments(&cs, &ss);
2814 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2815 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2816 ss_sel = cs_sel + 8;
2817 if (efer & EFER_LMA) {
2822 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2823 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2825 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2826 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2828 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2829 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2831 if (efer & EFER_LMA)
2832 ctxt->mode = X86EMUL_MODE_PROT64;
2834 return X86EMUL_CONTINUE;
2837 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2839 const struct x86_emulate_ops *ops = ctxt->ops;
2840 struct desc_struct cs, ss;
2841 u64 msr_data, rcx, rdx;
2843 u16 cs_sel = 0, ss_sel = 0;
2845 /* inject #GP if in real mode or Virtual 8086 mode */
2846 if (ctxt->mode == X86EMUL_MODE_REAL ||
2847 ctxt->mode == X86EMUL_MODE_VM86)
2848 return emulate_gp(ctxt, 0);
2850 setup_syscalls_segments(&cs, &ss);
2852 if ((ctxt->rex_prefix & 0x8) != 0x0)
2853 usermode = X86EMUL_MODE_PROT64;
2855 usermode = X86EMUL_MODE_PROT32;
2857 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2858 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2862 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2864 case X86EMUL_MODE_PROT32:
2865 cs_sel = (u16)(msr_data + 16);
2866 if ((msr_data & 0xfffc) == 0x0)
2867 return emulate_gp(ctxt, 0);
2868 ss_sel = (u16)(msr_data + 24);
2872 case X86EMUL_MODE_PROT64:
2873 cs_sel = (u16)(msr_data + 32);
2874 if (msr_data == 0x0)
2875 return emulate_gp(ctxt, 0);
2876 ss_sel = cs_sel + 8;
2879 if (emul_is_noncanonical_address(rcx, ctxt) ||
2880 emul_is_noncanonical_address(rdx, ctxt))
2881 return emulate_gp(ctxt, 0);
2884 cs_sel |= SEGMENT_RPL_MASK;
2885 ss_sel |= SEGMENT_RPL_MASK;
2887 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2888 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2891 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2893 return X86EMUL_CONTINUE;
2896 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2899 if (ctxt->mode == X86EMUL_MODE_REAL)
2901 if (ctxt->mode == X86EMUL_MODE_VM86)
2903 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2904 return ctxt->ops->cpl(ctxt) > iopl;
2907 #define VMWARE_PORT_VMPORT (0x5658)
2908 #define VMWARE_PORT_VMRPC (0x5659)
2910 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2913 const struct x86_emulate_ops *ops = ctxt->ops;
2914 struct desc_struct tr_seg;
2917 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2918 unsigned mask = (1 << len) - 1;
2922 * VMware allows access to these ports even if denied
2923 * by TSS I/O permission bitmap. Mimic behavior.
2925 if (enable_vmware_backdoor &&
2926 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2929 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2932 if (desc_limit_scaled(&tr_seg) < 103)
2934 base = get_desc_base(&tr_seg);
2935 #ifdef CONFIG_X86_64
2936 base |= ((u64)base3) << 32;
2938 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2939 if (r != X86EMUL_CONTINUE)
2941 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2943 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2944 if (r != X86EMUL_CONTINUE)
2946 if ((perm >> bit_idx) & mask)
2951 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2957 if (emulator_bad_iopl(ctxt))
2958 if (!emulator_io_port_access_allowed(ctxt, port, len))
2961 ctxt->perm_ok = true;
2966 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2969 * Intel CPUs mask the counter and pointers in quite strange
2970 * manner when ECX is zero due to REP-string optimizations.
2972 #ifdef CONFIG_X86_64
2973 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2976 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2979 case 0xa4: /* movsb */
2980 case 0xa5: /* movsd/w */
2981 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2983 case 0xaa: /* stosb */
2984 case 0xab: /* stosd/w */
2985 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2990 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2991 struct tss_segment_16 *tss)
2993 tss->ip = ctxt->_eip;
2994 tss->flag = ctxt->eflags;
2995 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2996 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2997 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2998 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2999 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3000 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3001 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3002 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3004 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3005 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3006 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3007 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3008 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3011 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3012 struct tss_segment_16 *tss)
3017 ctxt->_eip = tss->ip;
3018 ctxt->eflags = tss->flag | 2;
3019 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3020 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3021 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3022 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3023 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3024 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3025 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3026 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3029 * SDM says that segment selectors are loaded before segment
3032 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3033 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3034 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3035 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3036 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3041 * Now load segment descriptors. If fault happens at this stage
3042 * it is handled in a context of new task
3044 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3045 X86_TRANSFER_TASK_SWITCH, NULL);
3046 if (ret != X86EMUL_CONTINUE)
3048 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3049 X86_TRANSFER_TASK_SWITCH, NULL);
3050 if (ret != X86EMUL_CONTINUE)
3052 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3053 X86_TRANSFER_TASK_SWITCH, NULL);
3054 if (ret != X86EMUL_CONTINUE)
3056 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3057 X86_TRANSFER_TASK_SWITCH, NULL);
3058 if (ret != X86EMUL_CONTINUE)
3060 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3061 X86_TRANSFER_TASK_SWITCH, NULL);
3062 if (ret != X86EMUL_CONTINUE)
3065 return X86EMUL_CONTINUE;
3068 static int task_switch_16(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3069 ulong old_tss_base, struct desc_struct *new_desc)
3071 struct tss_segment_16 tss_seg;
3073 u32 new_tss_base = get_desc_base(new_desc);
3075 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3076 if (ret != X86EMUL_CONTINUE)
3079 save_state_to_tss16(ctxt, &tss_seg);
3081 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3082 if (ret != X86EMUL_CONTINUE)
3085 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3086 if (ret != X86EMUL_CONTINUE)
3089 if (old_tss_sel != 0xffff) {
3090 tss_seg.prev_task_link = old_tss_sel;
3092 ret = linear_write_system(ctxt, new_tss_base,
3093 &tss_seg.prev_task_link,
3094 sizeof(tss_seg.prev_task_link));
3095 if (ret != X86EMUL_CONTINUE)
3099 return load_state_from_tss16(ctxt, &tss_seg);
3102 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3103 struct tss_segment_32 *tss)
3105 /* CR3 and ldt selector are not saved intentionally */
3106 tss->eip = ctxt->_eip;
3107 tss->eflags = ctxt->eflags;
3108 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3109 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3110 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3111 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3112 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3113 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3114 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3115 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3117 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3118 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3119 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3120 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3121 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3122 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3125 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3126 struct tss_segment_32 *tss)
3131 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3132 return emulate_gp(ctxt, 0);
3133 ctxt->_eip = tss->eip;
3134 ctxt->eflags = tss->eflags | 2;
3136 /* General purpose registers */
3137 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3138 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3139 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3140 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3141 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3142 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3143 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3144 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3147 * SDM says that segment selectors are loaded before segment
3148 * descriptors. This is important because CPL checks will
3151 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3152 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3153 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3154 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3155 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3156 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3157 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3160 * If we're switching between Protected Mode and VM86, we need to make
3161 * sure to update the mode before loading the segment descriptors so
3162 * that the selectors are interpreted correctly.
3164 if (ctxt->eflags & X86_EFLAGS_VM) {
3165 ctxt->mode = X86EMUL_MODE_VM86;
3168 ctxt->mode = X86EMUL_MODE_PROT32;
3173 * Now load segment descriptors. If fault happens at this stage
3174 * it is handled in a context of new task
3176 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3177 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3178 if (ret != X86EMUL_CONTINUE)
3180 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3181 X86_TRANSFER_TASK_SWITCH, NULL);
3182 if (ret != X86EMUL_CONTINUE)
3184 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3185 X86_TRANSFER_TASK_SWITCH, NULL);
3186 if (ret != X86EMUL_CONTINUE)
3188 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3189 X86_TRANSFER_TASK_SWITCH, NULL);
3190 if (ret != X86EMUL_CONTINUE)
3192 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3193 X86_TRANSFER_TASK_SWITCH, NULL);
3194 if (ret != X86EMUL_CONTINUE)
3196 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3197 X86_TRANSFER_TASK_SWITCH, NULL);
3198 if (ret != X86EMUL_CONTINUE)
3200 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3201 X86_TRANSFER_TASK_SWITCH, NULL);
3206 static int task_switch_32(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3207 ulong old_tss_base, struct desc_struct *new_desc)
3209 struct tss_segment_32 tss_seg;
3211 u32 new_tss_base = get_desc_base(new_desc);
3212 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3213 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3215 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3216 if (ret != X86EMUL_CONTINUE)
3219 save_state_to_tss32(ctxt, &tss_seg);
3221 /* Only GP registers and segment selectors are saved */
3222 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3223 ldt_sel_offset - eip_offset);
3224 if (ret != X86EMUL_CONTINUE)
3227 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3228 if (ret != X86EMUL_CONTINUE)
3231 if (old_tss_sel != 0xffff) {
3232 tss_seg.prev_task_link = old_tss_sel;
3234 ret = linear_write_system(ctxt, new_tss_base,
3235 &tss_seg.prev_task_link,
3236 sizeof(tss_seg.prev_task_link));
3237 if (ret != X86EMUL_CONTINUE)
3241 return load_state_from_tss32(ctxt, &tss_seg);
3244 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3245 u16 tss_selector, int idt_index, int reason,
3246 bool has_error_code, u32 error_code)
3248 const struct x86_emulate_ops *ops = ctxt->ops;
3249 struct desc_struct curr_tss_desc, next_tss_desc;
3251 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3252 ulong old_tss_base =
3253 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3255 ulong desc_addr, dr7;
3257 /* FIXME: old_tss_base == ~0 ? */
3259 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3260 if (ret != X86EMUL_CONTINUE)
3262 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3263 if (ret != X86EMUL_CONTINUE)
3266 /* FIXME: check that next_tss_desc is tss */
3269 * Check privileges. The three cases are task switch caused by...
3271 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3272 * 2. Exception/IRQ/iret: No check is performed
3273 * 3. jmp/call to TSS/task-gate: No check is performed since the
3274 * hardware checks it before exiting.
3276 if (reason == TASK_SWITCH_GATE) {
3277 if (idt_index != -1) {
3278 /* Software interrupts */
3279 struct desc_struct task_gate_desc;
3282 ret = read_interrupt_descriptor(ctxt, idt_index,
3284 if (ret != X86EMUL_CONTINUE)
3287 dpl = task_gate_desc.dpl;
3288 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3289 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3293 desc_limit = desc_limit_scaled(&next_tss_desc);
3294 if (!next_tss_desc.p ||
3295 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3296 desc_limit < 0x2b)) {
3297 return emulate_ts(ctxt, tss_selector & 0xfffc);
3300 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3301 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3302 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3305 if (reason == TASK_SWITCH_IRET)
3306 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3308 /* set back link to prev task only if NT bit is set in eflags
3309 note that old_tss_sel is not used after this point */
3310 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3311 old_tss_sel = 0xffff;
3313 if (next_tss_desc.type & 8)
3314 ret = task_switch_32(ctxt, old_tss_sel, old_tss_base, &next_tss_desc);
3316 ret = task_switch_16(ctxt, old_tss_sel,
3317 old_tss_base, &next_tss_desc);
3318 if (ret != X86EMUL_CONTINUE)
3321 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3322 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3324 if (reason != TASK_SWITCH_IRET) {
3325 next_tss_desc.type |= (1 << 1); /* set busy flag */
3326 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3329 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3330 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3332 if (has_error_code) {
3333 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3334 ctxt->lock_prefix = 0;
3335 ctxt->src.val = (unsigned long) error_code;
3336 ret = em_push(ctxt);
3339 ops->get_dr(ctxt, 7, &dr7);
3340 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3345 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3346 u16 tss_selector, int idt_index, int reason,
3347 bool has_error_code, u32 error_code)
3351 invalidate_registers(ctxt);
3352 ctxt->_eip = ctxt->eip;
3353 ctxt->dst.type = OP_NONE;
3355 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3356 has_error_code, error_code);
3358 if (rc == X86EMUL_CONTINUE) {
3359 ctxt->eip = ctxt->_eip;
3360 writeback_registers(ctxt);
3363 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3366 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3369 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3371 register_address_increment(ctxt, reg, df * op->bytes);
3372 op->addr.mem.ea = register_address(ctxt, reg);
3375 static int em_das(struct x86_emulate_ctxt *ctxt)
3378 bool af, cf, old_cf;
3380 cf = ctxt->eflags & X86_EFLAGS_CF;
3386 af = ctxt->eflags & X86_EFLAGS_AF;
3387 if ((al & 0x0f) > 9 || af) {
3389 cf = old_cf | (al >= 250);
3394 if (old_al > 0x99 || old_cf) {
3400 /* Set PF, ZF, SF */
3401 ctxt->src.type = OP_IMM;
3403 ctxt->src.bytes = 1;
3404 fastop(ctxt, em_or);
3405 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3407 ctxt->eflags |= X86_EFLAGS_CF;
3409 ctxt->eflags |= X86_EFLAGS_AF;
3410 return X86EMUL_CONTINUE;
3413 static int em_aam(struct x86_emulate_ctxt *ctxt)
3417 if (ctxt->src.val == 0)
3418 return emulate_de(ctxt);
3420 al = ctxt->dst.val & 0xff;
3421 ah = al / ctxt->src.val;
3422 al %= ctxt->src.val;
3424 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3426 /* Set PF, ZF, SF */
3427 ctxt->src.type = OP_IMM;
3429 ctxt->src.bytes = 1;
3430 fastop(ctxt, em_or);
3432 return X86EMUL_CONTINUE;
3435 static int em_aad(struct x86_emulate_ctxt *ctxt)
3437 u8 al = ctxt->dst.val & 0xff;
3438 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3440 al = (al + (ah * ctxt->src.val)) & 0xff;
3442 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3444 /* Set PF, ZF, SF */
3445 ctxt->src.type = OP_IMM;
3447 ctxt->src.bytes = 1;
3448 fastop(ctxt, em_or);
3450 return X86EMUL_CONTINUE;
3453 static int em_call(struct x86_emulate_ctxt *ctxt)
3456 long rel = ctxt->src.val;
3458 ctxt->src.val = (unsigned long)ctxt->_eip;
3459 rc = jmp_rel(ctxt, rel);
3460 if (rc != X86EMUL_CONTINUE)
3462 return em_push(ctxt);
3465 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3470 struct desc_struct old_desc, new_desc;
3471 const struct x86_emulate_ops *ops = ctxt->ops;
3472 int cpl = ctxt->ops->cpl(ctxt);
3473 enum x86emul_mode prev_mode = ctxt->mode;
3475 old_eip = ctxt->_eip;
3476 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3478 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3479 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3480 X86_TRANSFER_CALL_JMP, &new_desc);
3481 if (rc != X86EMUL_CONTINUE)
3484 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3485 if (rc != X86EMUL_CONTINUE)
3488 ctxt->src.val = old_cs;
3490 if (rc != X86EMUL_CONTINUE)
3493 ctxt->src.val = old_eip;
3495 /* If we failed, we tainted the memory, but the very least we should
3497 if (rc != X86EMUL_CONTINUE) {
3498 pr_warn_once("faulting far call emulation tainted memory\n");
3503 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3504 ctxt->mode = prev_mode;
3509 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3514 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3515 if (rc != X86EMUL_CONTINUE)
3517 rc = assign_eip_near(ctxt, eip);
3518 if (rc != X86EMUL_CONTINUE)
3520 rsp_increment(ctxt, ctxt->src.val);
3521 return X86EMUL_CONTINUE;
3524 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3526 /* Write back the register source. */
3527 ctxt->src.val = ctxt->dst.val;
3528 write_register_operand(&ctxt->src);
3530 /* Write back the memory destination with implicit LOCK prefix. */
3531 ctxt->dst.val = ctxt->src.orig_val;
3532 ctxt->lock_prefix = 1;
3533 return X86EMUL_CONTINUE;
3536 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3538 ctxt->dst.val = ctxt->src2.val;
3539 return fastop(ctxt, em_imul);
3542 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3544 ctxt->dst.type = OP_REG;
3545 ctxt->dst.bytes = ctxt->src.bytes;
3546 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3547 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3549 return X86EMUL_CONTINUE;
3552 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3556 if (!ctxt->ops->guest_has_rdpid(ctxt))
3557 return emulate_ud(ctxt);
3559 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);
3560 ctxt->dst.val = tsc_aux;
3561 return X86EMUL_CONTINUE;
3564 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3568 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3569 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3570 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3571 return X86EMUL_CONTINUE;
3574 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3578 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3579 return emulate_gp(ctxt, 0);
3580 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3581 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3582 return X86EMUL_CONTINUE;
3585 static int em_mov(struct x86_emulate_ctxt *ctxt)
3587 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3588 return X86EMUL_CONTINUE;
3591 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3595 if (!ctxt->ops->guest_has_movbe(ctxt))
3596 return emulate_ud(ctxt);
3598 switch (ctxt->op_bytes) {
3601 * From MOVBE definition: "...When the operand size is 16 bits,
3602 * the upper word of the destination register remains unchanged
3605 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3606 * rules so we have to do the operation almost per hand.
3608 tmp = (u16)ctxt->src.val;
3609 ctxt->dst.val &= ~0xffffUL;
3610 ctxt->dst.val |= (unsigned long)swab16(tmp);
3613 ctxt->dst.val = swab32((u32)ctxt->src.val);
3616 ctxt->dst.val = swab64(ctxt->src.val);
3621 return X86EMUL_CONTINUE;
3624 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3626 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3627 return emulate_gp(ctxt, 0);
3629 /* Disable writeback. */
3630 ctxt->dst.type = OP_NONE;
3631 return X86EMUL_CONTINUE;
3634 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3638 if (ctxt->mode == X86EMUL_MODE_PROT64)
3639 val = ctxt->src.val & ~0ULL;
3641 val = ctxt->src.val & ~0U;
3643 /* #UD condition is already handled. */
3644 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3645 return emulate_gp(ctxt, 0);
3647 /* Disable writeback. */
3648 ctxt->dst.type = OP_NONE;
3649 return X86EMUL_CONTINUE;
3652 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3654 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3658 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3659 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3660 r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data);
3662 if (r == X86EMUL_IO_NEEDED)
3666 return emulate_gp(ctxt, 0);
3668 return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
3671 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3673 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3677 r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data);
3679 if (r == X86EMUL_IO_NEEDED)
3683 return emulate_gp(ctxt, 0);
3685 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3686 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3687 return X86EMUL_CONTINUE;
3690 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3692 if (segment > VCPU_SREG_GS &&
3693 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3694 ctxt->ops->cpl(ctxt) > 0)
3695 return emulate_gp(ctxt, 0);
3697 ctxt->dst.val = get_segment_selector(ctxt, segment);
3698 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3699 ctxt->dst.bytes = 2;
3700 return X86EMUL_CONTINUE;
3703 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3705 if (ctxt->modrm_reg > VCPU_SREG_GS)
3706 return emulate_ud(ctxt);
3708 return em_store_sreg(ctxt, ctxt->modrm_reg);
3711 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3713 u16 sel = ctxt->src.val;
3715 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3716 return emulate_ud(ctxt);
3718 if (ctxt->modrm_reg == VCPU_SREG_SS)
3719 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3721 /* Disable writeback. */
3722 ctxt->dst.type = OP_NONE;
3723 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3726 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3728 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3731 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3733 u16 sel = ctxt->src.val;
3735 /* Disable writeback. */
3736 ctxt->dst.type = OP_NONE;
3737 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3740 static int em_str(struct x86_emulate_ctxt *ctxt)
3742 return em_store_sreg(ctxt, VCPU_SREG_TR);
3745 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3747 u16 sel = ctxt->src.val;
3749 /* Disable writeback. */
3750 ctxt->dst.type = OP_NONE;
3751 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3754 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3759 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3760 if (rc == X86EMUL_CONTINUE)
3761 ctxt->ops->invlpg(ctxt, linear);
3762 /* Disable writeback. */
3763 ctxt->dst.type = OP_NONE;
3764 return X86EMUL_CONTINUE;
3767 static int em_clts(struct x86_emulate_ctxt *ctxt)
3771 cr0 = ctxt->ops->get_cr(ctxt, 0);
3773 ctxt->ops->set_cr(ctxt, 0, cr0);
3774 return X86EMUL_CONTINUE;
3777 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3779 int rc = ctxt->ops->fix_hypercall(ctxt);
3781 if (rc != X86EMUL_CONTINUE)
3784 /* Let the processor re-execute the fixed hypercall */
3785 ctxt->_eip = ctxt->eip;
3786 /* Disable writeback. */
3787 ctxt->dst.type = OP_NONE;
3788 return X86EMUL_CONTINUE;
3791 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3792 void (*get)(struct x86_emulate_ctxt *ctxt,
3793 struct desc_ptr *ptr))
3795 struct desc_ptr desc_ptr;
3797 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3798 ctxt->ops->cpl(ctxt) > 0)
3799 return emulate_gp(ctxt, 0);
3801 if (ctxt->mode == X86EMUL_MODE_PROT64)
3803 get(ctxt, &desc_ptr);
3804 if (ctxt->op_bytes == 2) {
3806 desc_ptr.address &= 0x00ffffff;
3808 /* Disable writeback. */
3809 ctxt->dst.type = OP_NONE;
3810 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3811 &desc_ptr, 2 + ctxt->op_bytes);
3814 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3816 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3819 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3821 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3824 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3826 struct desc_ptr desc_ptr;
3829 if (ctxt->mode == X86EMUL_MODE_PROT64)
3831 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3832 &desc_ptr.size, &desc_ptr.address,
3834 if (rc != X86EMUL_CONTINUE)
3836 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3837 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3838 return emulate_gp(ctxt, 0);
3840 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3842 ctxt->ops->set_idt(ctxt, &desc_ptr);
3843 /* Disable writeback. */
3844 ctxt->dst.type = OP_NONE;
3845 return X86EMUL_CONTINUE;
3848 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3850 return em_lgdt_lidt(ctxt, true);
3853 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3855 return em_lgdt_lidt(ctxt, false);
3858 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3860 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3861 ctxt->ops->cpl(ctxt) > 0)
3862 return emulate_gp(ctxt, 0);
3864 if (ctxt->dst.type == OP_MEM)
3865 ctxt->dst.bytes = 2;
3866 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3867 return X86EMUL_CONTINUE;
3870 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3872 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3873 | (ctxt->src.val & 0x0f));
3874 ctxt->dst.type = OP_NONE;
3875 return X86EMUL_CONTINUE;
3878 static int em_loop(struct x86_emulate_ctxt *ctxt)
3880 int rc = X86EMUL_CONTINUE;
3882 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3883 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3884 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3885 rc = jmp_rel(ctxt, ctxt->src.val);
3890 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3892 int rc = X86EMUL_CONTINUE;
3894 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3895 rc = jmp_rel(ctxt, ctxt->src.val);
3900 static int em_in(struct x86_emulate_ctxt *ctxt)
3902 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3904 return X86EMUL_IO_NEEDED;
3906 return X86EMUL_CONTINUE;
3909 static int em_out(struct x86_emulate_ctxt *ctxt)
3911 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3913 /* Disable writeback. */
3914 ctxt->dst.type = OP_NONE;
3915 return X86EMUL_CONTINUE;
3918 static int em_cli(struct x86_emulate_ctxt *ctxt)
3920 if (emulator_bad_iopl(ctxt))
3921 return emulate_gp(ctxt, 0);
3923 ctxt->eflags &= ~X86_EFLAGS_IF;
3924 return X86EMUL_CONTINUE;
3927 static int em_sti(struct x86_emulate_ctxt *ctxt)
3929 if (emulator_bad_iopl(ctxt))
3930 return emulate_gp(ctxt, 0);
3932 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3933 ctxt->eflags |= X86_EFLAGS_IF;
3934 return X86EMUL_CONTINUE;
3937 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3939 u32 eax, ebx, ecx, edx;
3942 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3943 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3944 ctxt->ops->cpl(ctxt)) {
3945 return emulate_gp(ctxt, 0);
3948 eax = reg_read(ctxt, VCPU_REGS_RAX);
3949 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3950 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3951 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3952 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3953 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3954 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3955 return X86EMUL_CONTINUE;
3958 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3962 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3964 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3966 ctxt->eflags &= ~0xffUL;
3967 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3968 return X86EMUL_CONTINUE;
3971 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3973 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3974 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3975 return X86EMUL_CONTINUE;
3978 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3980 switch (ctxt->op_bytes) {
3981 #ifdef CONFIG_X86_64
3983 asm("bswap %0" : "+r"(ctxt->dst.val));
3987 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3990 return X86EMUL_CONTINUE;
3993 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3995 /* emulating clflush regardless of cpuid */
3996 return X86EMUL_CONTINUE;
3999 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
4001 /* emulating clflushopt regardless of cpuid */
4002 return X86EMUL_CONTINUE;
4005 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4007 ctxt->dst.val = (s32) ctxt->src.val;
4008 return X86EMUL_CONTINUE;
4011 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4013 if (!ctxt->ops->guest_has_fxsr(ctxt))
4014 return emulate_ud(ctxt);
4016 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4017 return emulate_nm(ctxt);
4020 * Don't emulate a case that should never be hit, instead of working
4021 * around a lack of fxsave64/fxrstor64 on old compilers.
4023 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4024 return X86EMUL_UNHANDLEABLE;
4026 return X86EMUL_CONTINUE;
4030 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4031 * and restore MXCSR.
4033 static size_t __fxstate_size(int nregs)
4035 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4038 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4041 if (ctxt->mode == X86EMUL_MODE_PROT64)
4042 return __fxstate_size(16);
4044 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4045 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4049 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4052 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4053 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4055 * 3) 64-bit mode with REX.W prefix
4056 * - like (2), but XMM 8-15 are being saved and restored
4057 * 4) 64-bit mode without REX.W prefix
4058 * - like (3), but FIP and FDP are 64 bit
4060 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4061 * desired result. (4) is not emulated.
4063 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4064 * and FPU DS) should match.
4066 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4068 struct fxregs_state fx_state;
4071 rc = check_fxsr(ctxt);
4072 if (rc != X86EMUL_CONTINUE)
4077 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4081 if (rc != X86EMUL_CONTINUE)
4084 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4085 fxstate_size(ctxt));
4089 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4090 * in the host registers (via FXSAVE) instead, so they won't be modified.
4091 * (preemption has to stay disabled until FXRSTOR).
4093 * Use noinline to keep the stack for other functions called by callers small.
4095 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4096 const size_t used_size)
4098 struct fxregs_state fx_tmp;
4101 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4102 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4103 __fxstate_size(16) - used_size);
4108 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4110 struct fxregs_state fx_state;
4114 rc = check_fxsr(ctxt);
4115 if (rc != X86EMUL_CONTINUE)
4118 size = fxstate_size(ctxt);
4119 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4120 if (rc != X86EMUL_CONTINUE)
4125 if (size < __fxstate_size(16)) {
4126 rc = fxregs_fixup(&fx_state, size);
4127 if (rc != X86EMUL_CONTINUE)
4131 if (fx_state.mxcsr >> 16) {
4132 rc = emulate_gp(ctxt, 0);
4136 if (rc == X86EMUL_CONTINUE)
4137 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4145 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4149 eax = reg_read(ctxt, VCPU_REGS_RAX);
4150 edx = reg_read(ctxt, VCPU_REGS_RDX);
4151 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4153 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4154 return emulate_gp(ctxt, 0);
4156 return X86EMUL_CONTINUE;
4159 static bool valid_cr(int nr)
4171 static int check_cr_access(struct x86_emulate_ctxt *ctxt)
4173 if (!valid_cr(ctxt->modrm_reg))
4174 return emulate_ud(ctxt);
4176 return X86EMUL_CONTINUE;
4179 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4183 ctxt->ops->get_dr(ctxt, 7, &dr7);
4185 /* Check if DR7.Global_Enable is set */
4186 return dr7 & (1 << 13);
4189 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4191 int dr = ctxt->modrm_reg;
4195 return emulate_ud(ctxt);
4197 cr4 = ctxt->ops->get_cr(ctxt, 4);
4198 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4199 return emulate_ud(ctxt);
4201 if (check_dr7_gd(ctxt)) {
4204 ctxt->ops->get_dr(ctxt, 6, &dr6);
4205 dr6 &= ~DR_TRAP_BITS;
4206 dr6 |= DR6_BD | DR6_ACTIVE_LOW;
4207 ctxt->ops->set_dr(ctxt, 6, dr6);
4208 return emulate_db(ctxt);
4211 return X86EMUL_CONTINUE;
4214 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4216 u64 new_val = ctxt->src.val64;
4217 int dr = ctxt->modrm_reg;
4219 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4220 return emulate_gp(ctxt, 0);
4222 return check_dr_read(ctxt);
4225 static int check_svme(struct x86_emulate_ctxt *ctxt)
4229 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4231 if (!(efer & EFER_SVME))
4232 return emulate_ud(ctxt);
4234 return X86EMUL_CONTINUE;
4237 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4239 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4241 /* Valid physical address? */
4242 if (rax & 0xffff000000000000ULL)
4243 return emulate_gp(ctxt, 0);
4245 return check_svme(ctxt);
4248 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4250 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4252 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4253 return emulate_gp(ctxt, 0);
4255 return X86EMUL_CONTINUE;
4258 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4260 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4261 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4264 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4265 * in Ring3 when CR4.PCE=0.
4267 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4268 return X86EMUL_CONTINUE;
4271 * If CR4.PCE is set, the SDM requires CPL=0 or CR0.PE=0. The CR0.PE
4272 * check however is unnecessary because CPL is always 0 outside
4275 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4276 ctxt->ops->check_pmc(ctxt, rcx))
4277 return emulate_gp(ctxt, 0);
4279 return X86EMUL_CONTINUE;
4282 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4284 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4285 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4286 return emulate_gp(ctxt, 0);
4288 return X86EMUL_CONTINUE;
4291 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4293 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4294 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4295 return emulate_gp(ctxt, 0);
4297 return X86EMUL_CONTINUE;
4300 #define D(_y) { .flags = (_y) }
4301 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4302 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4303 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4304 #define N D(NotImpl)
4305 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4306 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4307 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4308 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4309 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4310 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4311 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4312 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4313 #define II(_f, _e, _i) \
4314 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4315 #define IIP(_f, _e, _i, _p) \
4316 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4317 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4318 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4320 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4321 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4322 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4323 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4324 #define I2bvIP(_f, _e, _i, _p) \
4325 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4327 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4328 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4329 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4331 static const struct opcode group7_rm0[] = {
4333 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4337 static const struct opcode group7_rm1[] = {
4338 DI(SrcNone | Priv, monitor),
4339 DI(SrcNone | Priv, mwait),
4343 static const struct opcode group7_rm2[] = {
4345 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4349 static const struct opcode group7_rm3[] = {
4350 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4351 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4352 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4353 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4354 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4355 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4356 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4357 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4360 static const struct opcode group7_rm7[] = {
4362 DIP(SrcNone, rdtscp, check_rdtsc),
4366 static const struct opcode group1[] = {
4368 F(Lock | PageTable, em_or),
4371 F(Lock | PageTable, em_and),
4377 static const struct opcode group1A[] = {
4378 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4381 static const struct opcode group2[] = {
4382 F(DstMem | ModRM, em_rol),
4383 F(DstMem | ModRM, em_ror),
4384 F(DstMem | ModRM, em_rcl),
4385 F(DstMem | ModRM, em_rcr),
4386 F(DstMem | ModRM, em_shl),
4387 F(DstMem | ModRM, em_shr),
4388 F(DstMem | ModRM, em_shl),
4389 F(DstMem | ModRM, em_sar),
4392 static const struct opcode group3[] = {
4393 F(DstMem | SrcImm | NoWrite, em_test),
4394 F(DstMem | SrcImm | NoWrite, em_test),
4395 F(DstMem | SrcNone | Lock, em_not),
4396 F(DstMem | SrcNone | Lock, em_neg),
4397 F(DstXacc | Src2Mem, em_mul_ex),
4398 F(DstXacc | Src2Mem, em_imul_ex),
4399 F(DstXacc | Src2Mem, em_div_ex),
4400 F(DstXacc | Src2Mem, em_idiv_ex),
4403 static const struct opcode group4[] = {
4404 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4405 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4409 static const struct opcode group5[] = {
4410 F(DstMem | SrcNone | Lock, em_inc),
4411 F(DstMem | SrcNone | Lock, em_dec),
4412 I(SrcMem | NearBranch | IsBranch, em_call_near_abs),
4413 I(SrcMemFAddr | ImplicitOps | IsBranch, em_call_far),
4414 I(SrcMem | NearBranch | IsBranch, em_jmp_abs),
4415 I(SrcMemFAddr | ImplicitOps | IsBranch, em_jmp_far),
4416 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4419 static const struct opcode group6[] = {
4420 II(Prot | DstMem, em_sldt, sldt),
4421 II(Prot | DstMem, em_str, str),
4422 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4423 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4427 static const struct group_dual group7 = { {
4428 II(Mov | DstMem, em_sgdt, sgdt),
4429 II(Mov | DstMem, em_sidt, sidt),
4430 II(SrcMem | Priv, em_lgdt, lgdt),
4431 II(SrcMem | Priv, em_lidt, lidt),
4432 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4433 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4434 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4440 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4441 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4445 static const struct opcode group8[] = {
4447 F(DstMem | SrcImmByte | NoWrite, em_bt),
4448 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4449 F(DstMem | SrcImmByte | Lock, em_btr),
4450 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4454 * The "memory" destination is actually always a register, since we come
4455 * from the register case of group9.
4457 static const struct gprefix pfx_0f_c7_7 = {
4458 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdpid),
4462 static const struct group_dual group9 = { {
4463 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4465 N, N, N, N, N, N, N,
4466 GP(0, &pfx_0f_c7_7),
4469 static const struct opcode group11[] = {
4470 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4474 static const struct gprefix pfx_0f_ae_7 = {
4475 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4478 static const struct group_dual group15 = { {
4479 I(ModRM | Aligned16, em_fxsave),
4480 I(ModRM | Aligned16, em_fxrstor),
4481 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4483 N, N, N, N, N, N, N, N,
4486 static const struct gprefix pfx_0f_6f_0f_7f = {
4487 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4490 static const struct instr_dual instr_dual_0f_2b = {
4494 static const struct gprefix pfx_0f_2b = {
4495 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4498 static const struct gprefix pfx_0f_10_0f_11 = {
4499 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4502 static const struct gprefix pfx_0f_28_0f_29 = {
4503 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4506 static const struct gprefix pfx_0f_e7 = {
4507 N, I(Sse, em_mov), N, N,
4510 static const struct escape escape_d9 = { {
4511 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4514 N, N, N, N, N, N, N, N,
4516 N, N, N, N, N, N, N, N,
4518 N, N, N, N, N, N, N, N,
4520 N, N, N, N, N, N, N, N,
4522 N, N, N, N, N, N, N, N,
4524 N, N, N, N, N, N, N, N,
4526 N, N, N, N, N, N, N, N,
4528 N, N, N, N, N, N, N, N,
4531 static const struct escape escape_db = { {
4532 N, N, N, N, N, N, N, N,
4535 N, N, N, N, N, N, N, N,
4537 N, N, N, N, N, N, N, N,
4539 N, N, N, N, N, N, N, N,
4541 N, N, N, N, N, N, N, N,
4543 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4545 N, N, N, N, N, N, N, N,
4547 N, N, N, N, N, N, N, N,
4549 N, N, N, N, N, N, N, N,
4552 static const struct escape escape_dd = { {
4553 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4556 N, N, N, N, N, N, N, N,
4558 N, N, N, N, N, N, N, N,
4560 N, N, N, N, N, N, N, N,
4562 N, N, N, N, N, N, N, N,
4564 N, N, N, N, N, N, N, N,
4566 N, N, N, N, N, N, N, N,
4568 N, N, N, N, N, N, N, N,
4570 N, N, N, N, N, N, N, N,
4573 static const struct instr_dual instr_dual_0f_c3 = {
4574 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4577 static const struct mode_dual mode_dual_63 = {
4578 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4581 static const struct instr_dual instr_dual_8d = {
4582 D(DstReg | SrcMem | ModRM | NoAccess), N
4585 static const struct opcode opcode_table[256] = {
4587 F6ALU(Lock, em_add),
4588 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4589 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4591 F6ALU(Lock | PageTable, em_or),
4592 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4595 F6ALU(Lock, em_adc),
4596 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4597 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4599 F6ALU(Lock, em_sbb),
4600 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4601 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4603 F6ALU(Lock | PageTable, em_and), N, N,
4605 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4607 F6ALU(Lock, em_xor), N, N,
4609 F6ALU(NoWrite, em_cmp), N, N,
4611 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4613 X8(I(SrcReg | Stack, em_push)),
4615 X8(I(DstReg | Stack, em_pop)),
4617 I(ImplicitOps | Stack | No64, em_pusha),
4618 I(ImplicitOps | Stack | No64, em_popa),
4619 N, MD(ModRM, &mode_dual_63),
4622 I(SrcImm | Mov | Stack, em_push),
4623 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4624 I(SrcImmByte | Mov | Stack, em_push),
4625 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4626 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4627 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4629 X16(D(SrcImmByte | NearBranch | IsBranch)),
4631 G(ByteOp | DstMem | SrcImm, group1),
4632 G(DstMem | SrcImm, group1),
4633 G(ByteOp | DstMem | SrcImm | No64, group1),
4634 G(DstMem | SrcImmByte, group1),
4635 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4636 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4638 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4639 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4640 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4641 ID(0, &instr_dual_8d),
4642 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4645 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4647 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4648 I(SrcImmFAddr | No64 | IsBranch, em_call_far), N,
4649 II(ImplicitOps | Stack, em_pushf, pushf),
4650 II(ImplicitOps | Stack, em_popf, popf),
4651 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4653 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4654 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4655 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4656 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4658 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4659 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4660 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4661 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4663 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4665 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4667 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4668 I(ImplicitOps | NearBranch | SrcImmU16 | IsBranch, em_ret_near_imm),
4669 I(ImplicitOps | NearBranch | IsBranch, em_ret),
4670 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4671 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4672 G(ByteOp, group11), G(0, group11),
4674 I(Stack | SrcImmU16 | Src2ImmByte | IsBranch, em_enter),
4675 I(Stack | IsBranch, em_leave),
4676 I(ImplicitOps | SrcImmU16 | IsBranch, em_ret_far_imm),
4677 I(ImplicitOps | IsBranch, em_ret_far),
4678 D(ImplicitOps | IsBranch), DI(SrcImmByte | IsBranch, intn),
4679 D(ImplicitOps | No64 | IsBranch),
4680 II(ImplicitOps | IsBranch, em_iret, iret),
4682 G(Src2One | ByteOp, group2), G(Src2One, group2),
4683 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4684 I(DstAcc | SrcImmUByte | No64, em_aam),
4685 I(DstAcc | SrcImmUByte | No64, em_aad),
4686 F(DstAcc | ByteOp | No64, em_salc),
4687 I(DstAcc | SrcXLat | ByteOp, em_mov),
4689 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4691 X3(I(SrcImmByte | NearBranch | IsBranch, em_loop)),
4692 I(SrcImmByte | NearBranch | IsBranch, em_jcxz),
4693 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4694 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4696 I(SrcImm | NearBranch | IsBranch, em_call),
4697 D(SrcImm | ImplicitOps | NearBranch | IsBranch),
4698 I(SrcImmFAddr | No64 | IsBranch, em_jmp_far),
4699 D(SrcImmByte | ImplicitOps | NearBranch | IsBranch),
4700 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4701 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4703 N, DI(ImplicitOps, icebp), N, N,
4704 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4705 G(ByteOp, group3), G(0, group3),
4707 D(ImplicitOps), D(ImplicitOps),
4708 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4709 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4712 static const struct opcode twobyte_table[256] = {
4714 G(0, group6), GD(0, &group7), N, N,
4715 N, I(ImplicitOps | EmulateOnUD | IsBranch, em_syscall),
4716 II(ImplicitOps | Priv, em_clts, clts), N,
4717 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4718 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4720 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4721 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4723 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4724 D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4725 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4726 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4727 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4728 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
4730 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access),
4731 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4732 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4734 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4737 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4738 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4739 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4742 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4743 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4744 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4745 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4746 I(ImplicitOps | EmulateOnUD | IsBranch, em_sysenter),
4747 I(ImplicitOps | Priv | EmulateOnUD | IsBranch, em_sysexit),
4749 N, N, N, N, N, N, N, N,
4751 X16(D(DstReg | SrcMem | ModRM)),
4753 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4758 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4763 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4765 X16(D(SrcImm | NearBranch | IsBranch)),
4767 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4769 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4770 II(ImplicitOps, em_cpuid, cpuid),
4771 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4772 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4773 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4775 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4776 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4777 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4778 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4779 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4780 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4782 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4783 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4784 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4785 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4786 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4787 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4791 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4792 I(DstReg | SrcMem | ModRM, em_bsf_c),
4793 I(DstReg | SrcMem | ModRM, em_bsr_c),
4794 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4796 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4797 N, ID(0, &instr_dual_0f_c3),
4798 N, N, N, GD(0, &group9),
4800 X8(I(DstReg, em_bswap)),
4802 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4804 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4805 N, N, N, N, N, N, N, N,
4807 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4810 static const struct instr_dual instr_dual_0f_38_f0 = {
4811 I(DstReg | SrcMem | Mov, em_movbe), N
4814 static const struct instr_dual instr_dual_0f_38_f1 = {
4815 I(DstMem | SrcReg | Mov, em_movbe), N
4818 static const struct gprefix three_byte_0f_38_f0 = {
4819 ID(0, &instr_dual_0f_38_f0), N, N, N
4822 static const struct gprefix three_byte_0f_38_f1 = {
4823 ID(0, &instr_dual_0f_38_f1), N, N, N
4827 * Insns below are selected by the prefix which indexed by the third opcode
4830 static const struct opcode opcode_map_0f_38[256] = {
4832 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4834 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4836 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4837 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4858 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4862 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4868 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4869 unsigned size, bool sign_extension)
4871 int rc = X86EMUL_CONTINUE;
4875 op->addr.mem.ea = ctxt->_eip;
4876 /* NB. Immediates are sign-extended as necessary. */
4877 switch (op->bytes) {
4879 op->val = insn_fetch(s8, ctxt);
4882 op->val = insn_fetch(s16, ctxt);
4885 op->val = insn_fetch(s32, ctxt);
4888 op->val = insn_fetch(s64, ctxt);
4891 if (!sign_extension) {
4892 switch (op->bytes) {
4900 op->val &= 0xffffffff;
4908 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4911 int rc = X86EMUL_CONTINUE;
4915 decode_register_operand(ctxt, op);
4918 rc = decode_imm(ctxt, op, 1, false);
4921 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4925 if (ctxt->d & BitOp)
4926 fetch_bit_operand(ctxt);
4927 op->orig_val = op->val;
4930 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4934 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4935 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4936 fetch_register_operand(op);
4937 op->orig_val = op->val;
4941 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4942 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4943 fetch_register_operand(op);
4944 op->orig_val = op->val;
4947 if (ctxt->d & ByteOp) {
4952 op->bytes = ctxt->op_bytes;
4953 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4954 fetch_register_operand(op);
4955 op->orig_val = op->val;
4959 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4961 register_address(ctxt, VCPU_REGS_RDI);
4962 op->addr.mem.seg = VCPU_SREG_ES;
4969 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4970 fetch_register_operand(op);
4975 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4978 rc = decode_imm(ctxt, op, 1, true);
4986 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4989 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4992 ctxt->memop.bytes = 1;
4993 if (ctxt->memop.type == OP_REG) {
4994 ctxt->memop.addr.reg = decode_register(ctxt,
4995 ctxt->modrm_rm, true);
4996 fetch_register_operand(&ctxt->memop);
5000 ctxt->memop.bytes = 2;
5003 ctxt->memop.bytes = 4;
5006 rc = decode_imm(ctxt, op, 2, false);
5009 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5013 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5015 register_address(ctxt, VCPU_REGS_RSI);
5016 op->addr.mem.seg = ctxt->seg_override;
5022 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5025 reg_read(ctxt, VCPU_REGS_RBX) +
5026 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5027 op->addr.mem.seg = ctxt->seg_override;
5032 op->addr.mem.ea = ctxt->_eip;
5033 op->bytes = ctxt->op_bytes + 2;
5034 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5037 ctxt->memop.bytes = ctxt->op_bytes + 2;
5041 op->val = VCPU_SREG_ES;
5045 op->val = VCPU_SREG_CS;
5049 op->val = VCPU_SREG_SS;
5053 op->val = VCPU_SREG_DS;
5057 op->val = VCPU_SREG_FS;
5061 op->val = VCPU_SREG_GS;
5064 /* Special instructions do their own operand decoding. */
5066 op->type = OP_NONE; /* Disable writeback. */
5074 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type)
5076 int rc = X86EMUL_CONTINUE;
5077 int mode = ctxt->mode;
5078 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5079 bool op_prefix = false;
5080 bool has_seg_override = false;
5081 struct opcode opcode;
5083 struct desc_struct desc;
5085 ctxt->memop.type = OP_NONE;
5086 ctxt->memopp = NULL;
5087 ctxt->_eip = ctxt->eip;
5088 ctxt->fetch.ptr = ctxt->fetch.data;
5089 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5090 ctxt->opcode_len = 1;
5091 ctxt->intercept = x86_intercept_none;
5093 memcpy(ctxt->fetch.data, insn, insn_len);
5095 rc = __do_insn_fetch_bytes(ctxt, 1);
5096 if (rc != X86EMUL_CONTINUE)
5101 case X86EMUL_MODE_REAL:
5102 case X86EMUL_MODE_VM86:
5103 def_op_bytes = def_ad_bytes = 2;
5104 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5106 def_op_bytes = def_ad_bytes = 4;
5108 case X86EMUL_MODE_PROT16:
5109 def_op_bytes = def_ad_bytes = 2;
5111 case X86EMUL_MODE_PROT32:
5112 def_op_bytes = def_ad_bytes = 4;
5114 #ifdef CONFIG_X86_64
5115 case X86EMUL_MODE_PROT64:
5121 return EMULATION_FAILED;
5124 ctxt->op_bytes = def_op_bytes;
5125 ctxt->ad_bytes = def_ad_bytes;
5127 /* Legacy prefixes. */
5129 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5130 case 0x66: /* operand-size override */
5132 /* switch between 2/4 bytes */
5133 ctxt->op_bytes = def_op_bytes ^ 6;
5135 case 0x67: /* address-size override */
5136 if (mode == X86EMUL_MODE_PROT64)
5137 /* switch between 4/8 bytes */
5138 ctxt->ad_bytes = def_ad_bytes ^ 12;
5140 /* switch between 2/4 bytes */
5141 ctxt->ad_bytes = def_ad_bytes ^ 6;
5143 case 0x26: /* ES override */
5144 has_seg_override = true;
5145 ctxt->seg_override = VCPU_SREG_ES;
5147 case 0x2e: /* CS override */
5148 has_seg_override = true;
5149 ctxt->seg_override = VCPU_SREG_CS;
5151 case 0x36: /* SS override */
5152 has_seg_override = true;
5153 ctxt->seg_override = VCPU_SREG_SS;
5155 case 0x3e: /* DS override */
5156 has_seg_override = true;
5157 ctxt->seg_override = VCPU_SREG_DS;
5159 case 0x64: /* FS override */
5160 has_seg_override = true;
5161 ctxt->seg_override = VCPU_SREG_FS;
5163 case 0x65: /* GS override */
5164 has_seg_override = true;
5165 ctxt->seg_override = VCPU_SREG_GS;
5167 case 0x40 ... 0x4f: /* REX */
5168 if (mode != X86EMUL_MODE_PROT64)
5170 ctxt->rex_prefix = ctxt->b;
5172 case 0xf0: /* LOCK */
5173 ctxt->lock_prefix = 1;
5175 case 0xf2: /* REPNE/REPNZ */
5176 case 0xf3: /* REP/REPE/REPZ */
5177 ctxt->rep_prefix = ctxt->b;
5183 /* Any legacy prefix after a REX prefix nullifies its effect. */
5185 ctxt->rex_prefix = 0;
5191 if (ctxt->rex_prefix & 8)
5192 ctxt->op_bytes = 8; /* REX.W */
5194 /* Opcode byte(s). */
5195 opcode = opcode_table[ctxt->b];
5196 /* Two-byte opcode? */
5197 if (ctxt->b == 0x0f) {
5198 ctxt->opcode_len = 2;
5199 ctxt->b = insn_fetch(u8, ctxt);
5200 opcode = twobyte_table[ctxt->b];
5202 /* 0F_38 opcode map */
5203 if (ctxt->b == 0x38) {
5204 ctxt->opcode_len = 3;
5205 ctxt->b = insn_fetch(u8, ctxt);
5206 opcode = opcode_map_0f_38[ctxt->b];
5209 ctxt->d = opcode.flags;
5211 if (ctxt->d & ModRM)
5212 ctxt->modrm = insn_fetch(u8, ctxt);
5214 /* vex-prefix instructions are not implemented */
5215 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5216 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5220 while (ctxt->d & GroupMask) {
5221 switch (ctxt->d & GroupMask) {
5223 goffset = (ctxt->modrm >> 3) & 7;
5224 opcode = opcode.u.group[goffset];
5227 goffset = (ctxt->modrm >> 3) & 7;
5228 if ((ctxt->modrm >> 6) == 3)
5229 opcode = opcode.u.gdual->mod3[goffset];
5231 opcode = opcode.u.gdual->mod012[goffset];
5234 goffset = ctxt->modrm & 7;
5235 opcode = opcode.u.group[goffset];
5238 if (ctxt->rep_prefix && op_prefix)
5239 return EMULATION_FAILED;
5240 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5241 switch (simd_prefix) {
5242 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5243 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5244 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5245 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5249 if (ctxt->modrm > 0xbf) {
5250 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5251 u32 index = array_index_nospec(
5252 ctxt->modrm - 0xc0, size);
5254 opcode = opcode.u.esc->high[index];
5256 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5260 if ((ctxt->modrm >> 6) == 3)
5261 opcode = opcode.u.idual->mod3;
5263 opcode = opcode.u.idual->mod012;
5266 if (ctxt->mode == X86EMUL_MODE_PROT64)
5267 opcode = opcode.u.mdual->mode64;
5269 opcode = opcode.u.mdual->mode32;
5272 return EMULATION_FAILED;
5275 ctxt->d &= ~(u64)GroupMask;
5276 ctxt->d |= opcode.flags;
5279 ctxt->is_branch = opcode.flags & IsBranch;
5283 return EMULATION_FAILED;
5285 ctxt->execute = opcode.u.execute;
5287 if (unlikely(emulation_type & EMULTYPE_TRAP_UD) &&
5288 likely(!(ctxt->d & EmulateOnUD)))
5289 return EMULATION_FAILED;
5291 if (unlikely(ctxt->d &
5292 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5295 * These are copied unconditionally here, and checked unconditionally
5296 * in x86_emulate_insn.
5298 ctxt->check_perm = opcode.check_perm;
5299 ctxt->intercept = opcode.intercept;
5301 if (ctxt->d & NotImpl)
5302 return EMULATION_FAILED;
5304 if (mode == X86EMUL_MODE_PROT64) {
5305 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5307 else if (ctxt->d & NearBranch)
5311 if (ctxt->d & Op3264) {
5312 if (mode == X86EMUL_MODE_PROT64)
5318 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5322 ctxt->op_bytes = 16;
5323 else if (ctxt->d & Mmx)
5327 /* ModRM and SIB bytes. */
5328 if (ctxt->d & ModRM) {
5329 rc = decode_modrm(ctxt, &ctxt->memop);
5330 if (!has_seg_override) {
5331 has_seg_override = true;
5332 ctxt->seg_override = ctxt->modrm_seg;
5334 } else if (ctxt->d & MemAbs)
5335 rc = decode_abs(ctxt, &ctxt->memop);
5336 if (rc != X86EMUL_CONTINUE)
5339 if (!has_seg_override)
5340 ctxt->seg_override = VCPU_SREG_DS;
5342 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5345 * Decode and fetch the source operand: register, memory
5348 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5349 if (rc != X86EMUL_CONTINUE)
5353 * Decode and fetch the second source operand: register, memory
5356 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5357 if (rc != X86EMUL_CONTINUE)
5360 /* Decode and fetch the destination operand: register or memory. */
5361 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5363 if (ctxt->rip_relative && likely(ctxt->memopp))
5364 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5365 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5368 if (rc == X86EMUL_PROPAGATE_FAULT)
5369 ctxt->have_exception = true;
5370 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5373 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5375 return ctxt->d & PageTable;
5378 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5380 /* The second termination condition only applies for REPE
5381 * and REPNE. Test if the repeat string operation prefix is
5382 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5383 * corresponding termination condition according to:
5384 * - if REPE/REPZ and ZF = 0 then done
5385 * - if REPNE/REPNZ and ZF = 1 then done
5387 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5388 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5389 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5390 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5391 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5392 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5398 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5403 rc = asm_safe("fwait");
5406 if (unlikely(rc != X86EMUL_CONTINUE))
5407 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5409 return X86EMUL_CONTINUE;
5412 static void fetch_possible_mmx_operand(struct operand *op)
5414 if (op->type == OP_MM)
5415 kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5418 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5420 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5422 if (!(ctxt->d & ByteOp))
5423 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5425 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5426 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5427 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5428 : "c"(ctxt->src2.val));
5430 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5431 if (!fop) /* exception is returned in fop variable */
5432 return emulate_de(ctxt);
5433 return X86EMUL_CONTINUE;
5436 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5438 /* Clear fields that are set conditionally but read without a guard. */
5439 ctxt->rip_relative = false;
5440 ctxt->rex_prefix = 0;
5441 ctxt->lock_prefix = 0;
5442 ctxt->rep_prefix = 0;
5443 ctxt->regs_valid = 0;
5444 ctxt->regs_dirty = 0;
5446 ctxt->io_read.pos = 0;
5447 ctxt->io_read.end = 0;
5448 ctxt->mem_read.end = 0;
5451 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5453 const struct x86_emulate_ops *ops = ctxt->ops;
5454 int rc = X86EMUL_CONTINUE;
5455 int saved_dst_type = ctxt->dst.type;
5456 unsigned emul_flags;
5458 ctxt->mem_read.pos = 0;
5460 /* LOCK prefix is allowed only with some instructions */
5461 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5462 rc = emulate_ud(ctxt);
5466 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5467 rc = emulate_ud(ctxt);
5471 emul_flags = ctxt->ops->get_hflags(ctxt);
5472 if (unlikely(ctxt->d &
5473 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5474 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5475 (ctxt->d & Undefined)) {
5476 rc = emulate_ud(ctxt);
5480 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5481 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5482 rc = emulate_ud(ctxt);
5486 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5487 rc = emulate_nm(ctxt);
5491 if (ctxt->d & Mmx) {
5492 rc = flush_pending_x87_faults(ctxt);
5493 if (rc != X86EMUL_CONTINUE)
5496 * Now that we know the fpu is exception safe, we can fetch
5499 fetch_possible_mmx_operand(&ctxt->src);
5500 fetch_possible_mmx_operand(&ctxt->src2);
5501 if (!(ctxt->d & Mov))
5502 fetch_possible_mmx_operand(&ctxt->dst);
5505 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5506 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5507 X86_ICPT_PRE_EXCEPT);
5508 if (rc != X86EMUL_CONTINUE)
5512 /* Instruction can only be executed in protected mode */
5513 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5514 rc = emulate_ud(ctxt);
5518 /* Privileged instruction can be executed only in CPL=0 */
5519 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5520 if (ctxt->d & PrivUD)
5521 rc = emulate_ud(ctxt);
5523 rc = emulate_gp(ctxt, 0);
5527 /* Do instruction specific permission checks */
5528 if (ctxt->d & CheckPerm) {
5529 rc = ctxt->check_perm(ctxt);
5530 if (rc != X86EMUL_CONTINUE)
5534 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5535 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5536 X86_ICPT_POST_EXCEPT);
5537 if (rc != X86EMUL_CONTINUE)
5541 if (ctxt->rep_prefix && (ctxt->d & String)) {
5542 /* All REP prefixes have the same first termination condition */
5543 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5544 string_registers_quirk(ctxt);
5545 ctxt->eip = ctxt->_eip;
5546 ctxt->eflags &= ~X86_EFLAGS_RF;
5552 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5553 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5554 ctxt->src.valptr, ctxt->src.bytes);
5555 if (rc != X86EMUL_CONTINUE)
5557 ctxt->src.orig_val64 = ctxt->src.val64;
5560 if (ctxt->src2.type == OP_MEM) {
5561 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5562 &ctxt->src2.val, ctxt->src2.bytes);
5563 if (rc != X86EMUL_CONTINUE)
5567 if ((ctxt->d & DstMask) == ImplicitOps)
5571 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5572 /* optimisation - avoid slow emulated read if Mov */
5573 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5574 &ctxt->dst.val, ctxt->dst.bytes);
5575 if (rc != X86EMUL_CONTINUE) {
5576 if (!(ctxt->d & NoWrite) &&
5577 rc == X86EMUL_PROPAGATE_FAULT &&
5578 ctxt->exception.vector == PF_VECTOR)
5579 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5583 /* Copy full 64-bit value for CMPXCHG8B. */
5584 ctxt->dst.orig_val64 = ctxt->dst.val64;
5588 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5589 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5590 X86_ICPT_POST_MEMACCESS);
5591 if (rc != X86EMUL_CONTINUE)
5595 if (ctxt->rep_prefix && (ctxt->d & String))
5596 ctxt->eflags |= X86_EFLAGS_RF;
5598 ctxt->eflags &= ~X86_EFLAGS_RF;
5600 if (ctxt->execute) {
5601 if (ctxt->d & Fastop)
5602 rc = fastop(ctxt, ctxt->fop);
5604 rc = ctxt->execute(ctxt);
5605 if (rc != X86EMUL_CONTINUE)
5610 if (ctxt->opcode_len == 2)
5612 else if (ctxt->opcode_len == 3)
5613 goto threebyte_insn;
5616 case 0x70 ... 0x7f: /* jcc (short) */
5617 if (test_cc(ctxt->b, ctxt->eflags))
5618 rc = jmp_rel(ctxt, ctxt->src.val);
5620 case 0x8d: /* lea r16/r32, m */
5621 ctxt->dst.val = ctxt->src.addr.mem.ea;
5623 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5624 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5625 ctxt->dst.type = OP_NONE;
5629 case 0x98: /* cbw/cwde/cdqe */
5630 switch (ctxt->op_bytes) {
5631 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5632 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5633 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5636 case 0xcc: /* int3 */
5637 rc = emulate_int(ctxt, 3);
5639 case 0xcd: /* int n */
5640 rc = emulate_int(ctxt, ctxt->src.val);
5642 case 0xce: /* into */
5643 if (ctxt->eflags & X86_EFLAGS_OF)
5644 rc = emulate_int(ctxt, 4);
5646 case 0xe9: /* jmp rel */
5647 case 0xeb: /* jmp rel short */
5648 rc = jmp_rel(ctxt, ctxt->src.val);
5649 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5651 case 0xf4: /* hlt */
5652 ctxt->ops->halt(ctxt);
5654 case 0xf5: /* cmc */
5655 /* complement carry flag from eflags reg */
5656 ctxt->eflags ^= X86_EFLAGS_CF;
5658 case 0xf8: /* clc */
5659 ctxt->eflags &= ~X86_EFLAGS_CF;
5661 case 0xf9: /* stc */
5662 ctxt->eflags |= X86_EFLAGS_CF;
5664 case 0xfc: /* cld */
5665 ctxt->eflags &= ~X86_EFLAGS_DF;
5667 case 0xfd: /* std */
5668 ctxt->eflags |= X86_EFLAGS_DF;
5671 goto cannot_emulate;
5674 if (rc != X86EMUL_CONTINUE)
5678 if (ctxt->d & SrcWrite) {
5679 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5680 rc = writeback(ctxt, &ctxt->src);
5681 if (rc != X86EMUL_CONTINUE)
5684 if (!(ctxt->d & NoWrite)) {
5685 rc = writeback(ctxt, &ctxt->dst);
5686 if (rc != X86EMUL_CONTINUE)
5691 * restore dst type in case the decoding will be reused
5692 * (happens for string instruction )
5694 ctxt->dst.type = saved_dst_type;
5696 if ((ctxt->d & SrcMask) == SrcSI)
5697 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5699 if ((ctxt->d & DstMask) == DstDI)
5700 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5702 if (ctxt->rep_prefix && (ctxt->d & String)) {
5704 struct read_cache *r = &ctxt->io_read;
5705 if ((ctxt->d & SrcMask) == SrcSI)
5706 count = ctxt->src.count;
5708 count = ctxt->dst.count;
5709 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5711 if (!string_insn_completed(ctxt)) {
5713 * Re-enter guest when pio read ahead buffer is empty
5714 * or, if it is not used, after each 1024 iteration.
5716 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5717 (r->end == 0 || r->end != r->pos)) {
5719 * Reset read cache. Usually happens before
5720 * decode, but since instruction is restarted
5721 * we have to do it here.
5723 ctxt->mem_read.end = 0;
5724 writeback_registers(ctxt);
5725 return EMULATION_RESTART;
5727 goto done; /* skip rip writeback */
5729 ctxt->eflags &= ~X86_EFLAGS_RF;
5732 ctxt->eip = ctxt->_eip;
5733 if (ctxt->mode != X86EMUL_MODE_PROT64)
5734 ctxt->eip = (u32)ctxt->_eip;
5737 if (rc == X86EMUL_PROPAGATE_FAULT) {
5738 if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt))
5739 return EMULATION_FAILED;
5740 ctxt->have_exception = true;
5742 if (rc == X86EMUL_INTERCEPTED)
5743 return EMULATION_INTERCEPTED;
5745 if (rc == X86EMUL_CONTINUE)
5746 writeback_registers(ctxt);
5748 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5752 case 0x09: /* wbinvd */
5753 (ctxt->ops->wbinvd)(ctxt);
5755 case 0x08: /* invd */
5756 case 0x0d: /* GrpP (prefetch) */
5757 case 0x18: /* Grp16 (prefetch/nop) */
5758 case 0x1f: /* nop */
5760 case 0x20: /* mov cr, reg */
5761 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5763 case 0x21: /* mov from dr to reg */
5764 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5766 case 0x40 ... 0x4f: /* cmov */
5767 if (test_cc(ctxt->b, ctxt->eflags))
5768 ctxt->dst.val = ctxt->src.val;
5769 else if (ctxt->op_bytes != 4)
5770 ctxt->dst.type = OP_NONE; /* no writeback */
5772 case 0x80 ... 0x8f: /* jnz rel, etc*/
5773 if (test_cc(ctxt->b, ctxt->eflags))
5774 rc = jmp_rel(ctxt, ctxt->src.val);
5776 case 0x90 ... 0x9f: /* setcc r/m8 */
5777 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5779 case 0xb6 ... 0xb7: /* movzx */
5780 ctxt->dst.bytes = ctxt->op_bytes;
5781 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5782 : (u16) ctxt->src.val;
5784 case 0xbe ... 0xbf: /* movsx */
5785 ctxt->dst.bytes = ctxt->op_bytes;
5786 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5787 (s16) ctxt->src.val;
5790 goto cannot_emulate;
5795 if (rc != X86EMUL_CONTINUE)
5801 return EMULATION_FAILED;
5804 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5806 invalidate_registers(ctxt);
5809 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5811 writeback_registers(ctxt);
5814 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5816 if (ctxt->rep_prefix && (ctxt->d & String))
5819 if (ctxt->d & TwoMemOp)