1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include "kvm_emulate.h"
24 #include <linux/stringify.h>
25 #include <asm/debugreg.h>
26 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
179 #define IsBranch ((u64)1 << 56) /* Instruction is considered a branch. */
181 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
183 #define X2(x...) x, x
184 #define X3(x...) X2(x), x
185 #define X4(x...) X2(x), X2(x)
186 #define X5(x...) X4(x), x
187 #define X6(x...) X4(x), X2(x)
188 #define X7(x...) X4(x), X3(x)
189 #define X8(x...) X4(x), X4(x)
190 #define X16(x...) X8(x), X8(x)
197 int (*execute)(struct x86_emulate_ctxt *ctxt);
198 const struct opcode *group;
199 const struct group_dual *gdual;
200 const struct gprefix *gprefix;
201 const struct escape *esc;
202 const struct instr_dual *idual;
203 const struct mode_dual *mdual;
204 void (*fastop)(struct fastop *fake);
206 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
210 struct opcode mod012[8];
211 struct opcode mod3[8];
215 struct opcode pfx_no;
216 struct opcode pfx_66;
217 struct opcode pfx_f2;
218 struct opcode pfx_f3;
223 struct opcode high[64];
227 struct opcode mod012;
232 struct opcode mode32;
233 struct opcode mode64;
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
238 enum x86_transfer_type {
240 X86_TRANSFER_CALL_JMP,
242 X86_TRANSFER_TASK_SWITCH,
245 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
247 if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))
248 nr &= NR_EMULATOR_GPRS - 1;
250 if (!(ctxt->regs_valid & (1 << nr))) {
251 ctxt->regs_valid |= 1 << nr;
252 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
254 return ctxt->_regs[nr];
257 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
259 if (KVM_EMULATOR_BUG_ON(nr >= NR_EMULATOR_GPRS, ctxt))
260 nr &= NR_EMULATOR_GPRS - 1;
262 BUILD_BUG_ON(sizeof(ctxt->regs_dirty) * BITS_PER_BYTE < NR_EMULATOR_GPRS);
263 BUILD_BUG_ON(sizeof(ctxt->regs_valid) * BITS_PER_BYTE < NR_EMULATOR_GPRS);
265 ctxt->regs_valid |= 1 << nr;
266 ctxt->regs_dirty |= 1 << nr;
267 return &ctxt->_regs[nr];
270 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
273 return reg_write(ctxt, nr);
276 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
278 unsigned long dirty = ctxt->regs_dirty;
281 for_each_set_bit(reg, &dirty, NR_EMULATOR_GPRS)
282 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
285 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
287 ctxt->regs_dirty = 0;
288 ctxt->regs_valid = 0;
292 * These EFLAGS bits are restored from saved value during emulation, and
293 * any changes are written back to the saved value after emulation.
295 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
296 X86_EFLAGS_PF|X86_EFLAGS_CF)
305 * fastop functions have a special calling convention:
310 * flags: rflags (in/out)
311 * ex: rsi (in:fastop pointer, out:zero if exception)
313 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
314 * different operand sizes can be reached by calculation, rather than a jump
315 * table (which would be bigger than the code).
317 * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR
318 * and 1 for the straight line speculation INT3, leaves 7 bytes for the
319 * body of the function. Currently none is larger than 4.
321 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
323 #define FASTOP_SIZE 16
325 #define __FOP_FUNC(name) \
326 ".align " __stringify(FASTOP_SIZE) " \n\t" \
327 ".type " name ", @function \n\t" \
332 #define FOP_FUNC(name) \
335 #define __FOP_RET(name) \
337 ".size " name ", .-" name "\n\t"
339 #define FOP_RET(name) \
342 #define __FOP_START(op, align) \
343 extern void em_##op(struct fastop *fake); \
344 asm(".pushsection .text, \"ax\" \n\t" \
345 ".global em_" #op " \n\t" \
346 ".align " __stringify(align) " \n\t" \
349 #define FOP_START(op) __FOP_START(op, FASTOP_SIZE)
354 #define __FOPNOP(name) \
359 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
361 #define FOP1E(op, dst) \
362 __FOP_FUNC(#op "_" #dst) \
363 "10: " #op " %" #dst " \n\t" \
364 __FOP_RET(#op "_" #dst)
366 #define FOP1EEX(op, dst) \
367 FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
369 #define FASTOP1(op) \
374 ON64(FOP1E(op##q, rax)) \
377 /* 1-operand, using src2 (for MUL/DIV r/m) */
378 #define FASTOP1SRC2(op, name) \
383 ON64(FOP1E(op, rcx)) \
386 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
387 #define FASTOP1SRC2EX(op, name) \
392 ON64(FOP1EEX(op, rcx)) \
395 #define FOP2E(op, dst, src) \
396 __FOP_FUNC(#op "_" #dst "_" #src) \
397 #op " %" #src ", %" #dst " \n\t" \
398 __FOP_RET(#op "_" #dst "_" #src)
400 #define FASTOP2(op) \
402 FOP2E(op##b, al, dl) \
403 FOP2E(op##w, ax, dx) \
404 FOP2E(op##l, eax, edx) \
405 ON64(FOP2E(op##q, rax, rdx)) \
408 /* 2 operand, word only */
409 #define FASTOP2W(op) \
412 FOP2E(op##w, ax, dx) \
413 FOP2E(op##l, eax, edx) \
414 ON64(FOP2E(op##q, rax, rdx)) \
417 /* 2 operand, src is CL */
418 #define FASTOP2CL(op) \
420 FOP2E(op##b, al, cl) \
421 FOP2E(op##w, ax, cl) \
422 FOP2E(op##l, eax, cl) \
423 ON64(FOP2E(op##q, rax, cl)) \
426 /* 2 operand, src and dest are reversed */
427 #define FASTOP2R(op, name) \
429 FOP2E(op##b, dl, al) \
430 FOP2E(op##w, dx, ax) \
431 FOP2E(op##l, edx, eax) \
432 ON64(FOP2E(op##q, rdx, rax)) \
435 #define FOP3E(op, dst, src, src2) \
436 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
437 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
438 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
440 /* 3-operand, word-only, src2=cl */
441 #define FASTOP3WCL(op) \
444 FOP3E(op##w, ax, dx, cl) \
445 FOP3E(op##l, eax, edx, cl) \
446 ON64(FOP3E(op##q, rax, rdx, cl)) \
449 /* Special case for SETcc - 1 instruction per cc */
450 #define FOP_SETCC(op) \
476 "pushf; sbb %al, %al; popf \n\t"
481 * XXX: inoutclob user must know where the argument is being expanded.
482 * Using asm goto would allow us to remove _fault.
484 #define asm_safe(insn, inoutclob...) \
488 asm volatile("1:" insn "\n" \
490 _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[_fault]) \
491 : [_fault] "+r"(_fault) inoutclob ); \
493 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
496 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
497 enum x86_intercept intercept,
498 enum x86_intercept_stage stage)
500 struct x86_instruction_info info = {
501 .intercept = intercept,
502 .rep_prefix = ctxt->rep_prefix,
503 .modrm_mod = ctxt->modrm_mod,
504 .modrm_reg = ctxt->modrm_reg,
505 .modrm_rm = ctxt->modrm_rm,
506 .src_val = ctxt->src.val64,
507 .dst_val = ctxt->dst.val64,
508 .src_bytes = ctxt->src.bytes,
509 .dst_bytes = ctxt->dst.bytes,
510 .ad_bytes = ctxt->ad_bytes,
511 .next_rip = ctxt->eip,
514 return ctxt->ops->intercept(ctxt, &info, stage);
517 static void assign_masked(ulong *dest, ulong src, ulong mask)
519 *dest = (*dest & ~mask) | (src & mask);
522 static void assign_register(unsigned long *reg, u64 val, int bytes)
524 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
527 *(u8 *)reg = (u8)val;
530 *(u16 *)reg = (u16)val;
534 break; /* 64b: zero-extend */
541 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
543 return (1UL << (ctxt->ad_bytes << 3)) - 1;
546 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
549 struct desc_struct ss;
551 if (ctxt->mode == X86EMUL_MODE_PROT64)
553 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
554 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
557 static int stack_size(struct x86_emulate_ctxt *ctxt)
559 return (__fls(stack_mask(ctxt)) + 1) >> 3;
562 /* Access/update address held in a register, based on addressing mode. */
563 static inline unsigned long
564 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
566 if (ctxt->ad_bytes == sizeof(unsigned long))
569 return reg & ad_mask(ctxt);
572 static inline unsigned long
573 register_address(struct x86_emulate_ctxt *ctxt, int reg)
575 return address_mask(ctxt, reg_read(ctxt, reg));
578 static void masked_increment(ulong *reg, ulong mask, int inc)
580 assign_masked(reg, *reg + inc, mask);
584 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
586 ulong *preg = reg_rmw(ctxt, reg);
588 assign_register(preg, *preg + inc, ctxt->ad_bytes);
591 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
593 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
596 static u32 desc_limit_scaled(struct desc_struct *desc)
598 u32 limit = get_desc_limit(desc);
600 return desc->g ? (limit << 12) | 0xfff : limit;
603 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
605 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
608 return ctxt->ops->get_cached_segment_base(ctxt, seg);
611 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
612 u32 error, bool valid)
614 if (KVM_EMULATOR_BUG_ON(vec > 0x1f, ctxt))
615 return X86EMUL_UNHANDLEABLE;
617 ctxt->exception.vector = vec;
618 ctxt->exception.error_code = error;
619 ctxt->exception.error_code_valid = valid;
620 return X86EMUL_PROPAGATE_FAULT;
623 static int emulate_db(struct x86_emulate_ctxt *ctxt)
625 return emulate_exception(ctxt, DB_VECTOR, 0, false);
628 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
630 return emulate_exception(ctxt, GP_VECTOR, err, true);
633 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
635 return emulate_exception(ctxt, SS_VECTOR, err, true);
638 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
640 return emulate_exception(ctxt, UD_VECTOR, 0, false);
643 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
645 return emulate_exception(ctxt, TS_VECTOR, err, true);
648 static int emulate_de(struct x86_emulate_ctxt *ctxt)
650 return emulate_exception(ctxt, DE_VECTOR, 0, false);
653 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
655 return emulate_exception(ctxt, NM_VECTOR, 0, false);
658 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
661 struct desc_struct desc;
663 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
667 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
672 struct desc_struct desc;
674 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
675 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
678 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
680 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
683 static inline bool emul_is_noncanonical_address(u64 la,
684 struct x86_emulate_ctxt *ctxt)
686 return !__is_canonical_address(la, ctxt_virt_addr_bits(ctxt));
690 * x86 defines three classes of vector instructions: explicitly
691 * aligned, explicitly unaligned, and the rest, which change behaviour
692 * depending on whether they're AVX encoded or not.
694 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
695 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
696 * 512 bytes of data must be aligned to a 16 byte boundary.
698 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
700 u64 alignment = ctxt->d & AlignMask;
702 if (likely(size < 16))
717 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
718 struct segmented_address addr,
719 unsigned *max_size, unsigned size,
720 bool write, bool fetch,
721 enum x86emul_mode mode, ulong *linear)
723 struct desc_struct desc;
730 la = seg_base(ctxt, addr.seg) + addr.ea;
733 case X86EMUL_MODE_PROT64:
735 va_bits = ctxt_virt_addr_bits(ctxt);
736 if (!__is_canonical_address(la, va_bits))
739 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
740 if (size > *max_size)
744 *linear = la = (u32)la;
745 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
749 /* code segment in protected mode or read-only data segment */
750 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
751 || !(desc.type & 2)) && write)
753 /* unreadable code segment */
754 if (!fetch && (desc.type & 8) && !(desc.type & 2))
756 lim = desc_limit_scaled(&desc);
757 if (!(desc.type & 8) && (desc.type & 4)) {
758 /* expand-down segment */
761 lim = desc.d ? 0xffffffff : 0xffff;
765 if (lim == 0xffffffff)
768 *max_size = (u64)lim + 1 - addr.ea;
769 if (size > *max_size)
774 if (la & (insn_alignment(ctxt, size) - 1))
775 return emulate_gp(ctxt, 0);
776 return X86EMUL_CONTINUE;
778 if (addr.seg == VCPU_SREG_SS)
779 return emulate_ss(ctxt, 0);
781 return emulate_gp(ctxt, 0);
784 static int linearize(struct x86_emulate_ctxt *ctxt,
785 struct segmented_address addr,
786 unsigned size, bool write,
790 return __linearize(ctxt, addr, &max_size, size, write, false,
794 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
795 enum x86emul_mode mode)
800 struct segmented_address addr = { .seg = VCPU_SREG_CS,
803 if (ctxt->op_bytes != sizeof(unsigned long))
804 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
805 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
806 if (rc == X86EMUL_CONTINUE)
807 ctxt->_eip = addr.ea;
811 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
813 return assign_eip(ctxt, dst, ctxt->mode);
816 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
817 const struct desc_struct *cs_desc)
819 enum x86emul_mode mode = ctxt->mode;
823 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
827 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
829 mode = X86EMUL_MODE_PROT64;
831 mode = X86EMUL_MODE_PROT32; /* temporary value */
834 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
835 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
836 rc = assign_eip(ctxt, dst, mode);
837 if (rc == X86EMUL_CONTINUE)
842 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
844 return assign_eip_near(ctxt, ctxt->_eip + rel);
847 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
848 void *data, unsigned size)
850 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
853 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
854 ulong linear, void *data,
857 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
860 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
861 struct segmented_address addr,
868 rc = linearize(ctxt, addr, size, false, &linear);
869 if (rc != X86EMUL_CONTINUE)
871 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
874 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
875 struct segmented_address addr,
882 rc = linearize(ctxt, addr, size, true, &linear);
883 if (rc != X86EMUL_CONTINUE)
885 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
889 * Prefetch the remaining bytes of the instruction without crossing page
890 * boundary if they are not in fetch_cache yet.
892 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
895 unsigned size, max_size;
896 unsigned long linear;
897 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
898 struct segmented_address addr = { .seg = VCPU_SREG_CS,
899 .ea = ctxt->eip + cur_size };
902 * We do not know exactly how many bytes will be needed, and
903 * __linearize is expensive, so fetch as much as possible. We
904 * just have to avoid going beyond the 15 byte limit, the end
905 * of the segment, or the end of the page.
907 * __linearize is called with size 0 so that it does not do any
908 * boundary check itself. Instead, we use max_size to check
911 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
913 if (unlikely(rc != X86EMUL_CONTINUE))
916 size = min_t(unsigned, 15UL ^ cur_size, max_size);
917 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
920 * One instruction can only straddle two pages,
921 * and one has been loaded at the beginning of
922 * x86_decode_insn. So, if not enough bytes
923 * still, we must have hit the 15-byte boundary.
925 if (unlikely(size < op_size))
926 return emulate_gp(ctxt, 0);
928 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
929 size, &ctxt->exception);
930 if (unlikely(rc != X86EMUL_CONTINUE))
932 ctxt->fetch.end += size;
933 return X86EMUL_CONTINUE;
936 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
939 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
941 if (unlikely(done_size < size))
942 return __do_insn_fetch_bytes(ctxt, size - done_size);
944 return X86EMUL_CONTINUE;
947 /* Fetch next part of the instruction being emulated. */
948 #define insn_fetch(_type, _ctxt) \
951 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
952 if (rc != X86EMUL_CONTINUE) \
954 ctxt->_eip += sizeof(_type); \
955 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
956 ctxt->fetch.ptr += sizeof(_type); \
960 #define insn_fetch_arr(_arr, _size, _ctxt) \
962 rc = do_insn_fetch_bytes(_ctxt, _size); \
963 if (rc != X86EMUL_CONTINUE) \
965 ctxt->_eip += (_size); \
966 memcpy(_arr, ctxt->fetch.ptr, _size); \
967 ctxt->fetch.ptr += (_size); \
971 * Given the 'reg' portion of a ModRM byte, and a register block, return a
972 * pointer into the block that addresses the relevant register.
973 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
975 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
979 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
981 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
982 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
984 p = reg_rmw(ctxt, modrm_reg);
988 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
989 struct segmented_address addr,
990 u16 *size, unsigned long *address, int op_bytes)
997 rc = segmented_read_std(ctxt, addr, size, 2);
998 if (rc != X86EMUL_CONTINUE)
1001 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1015 FASTOP1SRC2(mul, mul_ex);
1016 FASTOP1SRC2(imul, imul_ex);
1017 FASTOP1SRC2EX(div, div_ex);
1018 FASTOP1SRC2EX(idiv, idiv_ex);
1047 FASTOP2R(cmp, cmp_r);
1049 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1051 /* If src is zero, do not writeback, but update flags */
1052 if (ctxt->src.val == 0)
1053 ctxt->dst.type = OP_NONE;
1054 return fastop(ctxt, em_bsf);
1057 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1059 /* If src is zero, do not writeback, but update flags */
1060 if (ctxt->src.val == 0)
1061 ctxt->dst.type = OP_NONE;
1062 return fastop(ctxt, em_bsr);
1065 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1068 void (*fop)(void) = (void *)em_setcc + FASTOP_SIZE * (condition & 0xf);
1070 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1071 asm("push %[flags]; popf; " CALL_NOSPEC
1072 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1076 static void fetch_register_operand(struct operand *op)
1078 switch (op->bytes) {
1080 op->val = *(u8 *)op->addr.reg;
1083 op->val = *(u16 *)op->addr.reg;
1086 op->val = *(u32 *)op->addr.reg;
1089 op->val = *(u64 *)op->addr.reg;
1094 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1096 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1097 return emulate_nm(ctxt);
1100 asm volatile("fninit");
1102 return X86EMUL_CONTINUE;
1105 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1109 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1110 return emulate_nm(ctxt);
1113 asm volatile("fnstcw %0": "+m"(fcw));
1116 ctxt->dst.val = fcw;
1118 return X86EMUL_CONTINUE;
1121 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1125 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1126 return emulate_nm(ctxt);
1129 asm volatile("fnstsw %0": "+m"(fsw));
1132 ctxt->dst.val = fsw;
1134 return X86EMUL_CONTINUE;
1137 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1140 unsigned reg = ctxt->modrm_reg;
1142 if (!(ctxt->d & ModRM))
1143 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1145 if (ctxt->d & Sse) {
1149 kvm_read_sse_reg(reg, &op->vec_val);
1152 if (ctxt->d & Mmx) {
1161 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1162 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1164 fetch_register_operand(op);
1165 op->orig_val = op->val;
1168 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1170 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1171 ctxt->modrm_seg = VCPU_SREG_SS;
1174 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1178 int index_reg, base_reg, scale;
1179 int rc = X86EMUL_CONTINUE;
1182 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1183 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1184 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1186 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1187 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1188 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1189 ctxt->modrm_seg = VCPU_SREG_DS;
1191 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1193 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1194 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1196 if (ctxt->d & Sse) {
1199 op->addr.xmm = ctxt->modrm_rm;
1200 kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);
1203 if (ctxt->d & Mmx) {
1206 op->addr.mm = ctxt->modrm_rm & 7;
1209 fetch_register_operand(op);
1215 if (ctxt->ad_bytes == 2) {
1216 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1217 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1218 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1219 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1221 /* 16-bit ModR/M decode. */
1222 switch (ctxt->modrm_mod) {
1224 if (ctxt->modrm_rm == 6)
1225 modrm_ea += insn_fetch(u16, ctxt);
1228 modrm_ea += insn_fetch(s8, ctxt);
1231 modrm_ea += insn_fetch(u16, ctxt);
1234 switch (ctxt->modrm_rm) {
1236 modrm_ea += bx + si;
1239 modrm_ea += bx + di;
1242 modrm_ea += bp + si;
1245 modrm_ea += bp + di;
1254 if (ctxt->modrm_mod != 0)
1261 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1262 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1263 ctxt->modrm_seg = VCPU_SREG_SS;
1264 modrm_ea = (u16)modrm_ea;
1266 /* 32/64-bit ModR/M decode. */
1267 if ((ctxt->modrm_rm & 7) == 4) {
1268 sib = insn_fetch(u8, ctxt);
1269 index_reg |= (sib >> 3) & 7;
1270 base_reg |= sib & 7;
1273 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1274 modrm_ea += insn_fetch(s32, ctxt);
1276 modrm_ea += reg_read(ctxt, base_reg);
1277 adjust_modrm_seg(ctxt, base_reg);
1278 /* Increment ESP on POP [ESP] */
1279 if ((ctxt->d & IncSP) &&
1280 base_reg == VCPU_REGS_RSP)
1281 modrm_ea += ctxt->op_bytes;
1284 modrm_ea += reg_read(ctxt, index_reg) << scale;
1285 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1286 modrm_ea += insn_fetch(s32, ctxt);
1287 if (ctxt->mode == X86EMUL_MODE_PROT64)
1288 ctxt->rip_relative = 1;
1290 base_reg = ctxt->modrm_rm;
1291 modrm_ea += reg_read(ctxt, base_reg);
1292 adjust_modrm_seg(ctxt, base_reg);
1294 switch (ctxt->modrm_mod) {
1296 modrm_ea += insn_fetch(s8, ctxt);
1299 modrm_ea += insn_fetch(s32, ctxt);
1303 op->addr.mem.ea = modrm_ea;
1304 if (ctxt->ad_bytes != 8)
1305 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1311 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1314 int rc = X86EMUL_CONTINUE;
1317 switch (ctxt->ad_bytes) {
1319 op->addr.mem.ea = insn_fetch(u16, ctxt);
1322 op->addr.mem.ea = insn_fetch(u32, ctxt);
1325 op->addr.mem.ea = insn_fetch(u64, ctxt);
1332 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1336 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1337 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1339 if (ctxt->src.bytes == 2)
1340 sv = (s16)ctxt->src.val & (s16)mask;
1341 else if (ctxt->src.bytes == 4)
1342 sv = (s32)ctxt->src.val & (s32)mask;
1344 sv = (s64)ctxt->src.val & (s64)mask;
1346 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1347 ctxt->dst.addr.mem.ea + (sv >> 3));
1350 /* only subword offset */
1351 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1354 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1355 unsigned long addr, void *dest, unsigned size)
1358 struct read_cache *mc = &ctxt->mem_read;
1360 if (mc->pos < mc->end)
1363 if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))
1364 return X86EMUL_UNHANDLEABLE;
1366 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1368 if (rc != X86EMUL_CONTINUE)
1374 memcpy(dest, mc->data + mc->pos, size);
1376 return X86EMUL_CONTINUE;
1379 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1380 struct segmented_address addr,
1387 rc = linearize(ctxt, addr, size, false, &linear);
1388 if (rc != X86EMUL_CONTINUE)
1390 return read_emulated(ctxt, linear, data, size);
1393 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1394 struct segmented_address addr,
1401 rc = linearize(ctxt, addr, size, true, &linear);
1402 if (rc != X86EMUL_CONTINUE)
1404 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1408 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1409 struct segmented_address addr,
1410 const void *orig_data, const void *data,
1416 rc = linearize(ctxt, addr, size, true, &linear);
1417 if (rc != X86EMUL_CONTINUE)
1419 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1420 size, &ctxt->exception);
1423 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1424 unsigned int size, unsigned short port,
1427 struct read_cache *rc = &ctxt->io_read;
1429 if (rc->pos == rc->end) { /* refill pio read ahead */
1430 unsigned int in_page, n;
1431 unsigned int count = ctxt->rep_prefix ?
1432 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1433 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1434 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1435 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1436 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1439 rc->pos = rc->end = 0;
1440 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1445 if (ctxt->rep_prefix && (ctxt->d & String) &&
1446 !(ctxt->eflags & X86_EFLAGS_DF)) {
1447 ctxt->dst.data = rc->data + rc->pos;
1448 ctxt->dst.type = OP_MEM_STR;
1449 ctxt->dst.count = (rc->end - rc->pos) / size;
1452 memcpy(dest, rc->data + rc->pos, size);
1458 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1459 u16 index, struct desc_struct *desc)
1464 ctxt->ops->get_idt(ctxt, &dt);
1466 if (dt.size < index * 8 + 7)
1467 return emulate_gp(ctxt, index << 3 | 0x2);
1469 addr = dt.address + index * 8;
1470 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1473 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1474 u16 selector, struct desc_ptr *dt)
1476 const struct x86_emulate_ops *ops = ctxt->ops;
1479 if (selector & 1 << 2) {
1480 struct desc_struct desc;
1483 memset(dt, 0, sizeof(*dt));
1484 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1488 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1489 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1491 ops->get_gdt(ctxt, dt);
1494 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1495 u16 selector, ulong *desc_addr_p)
1498 u16 index = selector >> 3;
1501 get_descriptor_table_ptr(ctxt, selector, &dt);
1503 if (dt.size < index * 8 + 7)
1504 return emulate_gp(ctxt, selector & 0xfffc);
1506 addr = dt.address + index * 8;
1508 #ifdef CONFIG_X86_64
1509 if (addr >> 32 != 0) {
1512 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1513 if (!(efer & EFER_LMA))
1518 *desc_addr_p = addr;
1519 return X86EMUL_CONTINUE;
1522 /* allowed just for 8 bytes segments */
1523 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1524 u16 selector, struct desc_struct *desc,
1529 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1530 if (rc != X86EMUL_CONTINUE)
1533 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1536 /* allowed just for 8 bytes segments */
1537 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1538 u16 selector, struct desc_struct *desc)
1543 rc = get_descriptor_ptr(ctxt, selector, &addr);
1544 if (rc != X86EMUL_CONTINUE)
1547 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1550 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1551 u16 selector, int seg, u8 cpl,
1552 enum x86_transfer_type transfer,
1553 struct desc_struct *desc)
1555 struct desc_struct seg_desc, old_desc;
1557 unsigned err_vec = GP_VECTOR;
1559 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1565 memset(&seg_desc, 0, sizeof(seg_desc));
1567 if (ctxt->mode == X86EMUL_MODE_REAL) {
1568 /* set real mode segment descriptor (keep limit etc. for
1570 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1571 set_desc_base(&seg_desc, selector << 4);
1573 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1574 /* VM86 needs a clean new segment descriptor */
1575 set_desc_base(&seg_desc, selector << 4);
1576 set_desc_limit(&seg_desc, 0xffff);
1586 /* TR should be in GDT only */
1587 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1590 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1591 if (null_selector) {
1592 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1595 if (seg == VCPU_SREG_SS) {
1596 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1600 * ctxt->ops->set_segment expects the CPL to be in
1601 * SS.DPL, so fake an expand-up 32-bit data segment.
1611 /* Skip all following checks */
1615 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1616 if (ret != X86EMUL_CONTINUE)
1619 err_code = selector & 0xfffc;
1620 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1623 /* can't load system descriptor into segment selector */
1624 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1625 if (transfer == X86_TRANSFER_CALL_JMP)
1626 return X86EMUL_UNHANDLEABLE;
1635 * segment is not a writable data segment or segment
1636 * selector's RPL != CPL or segment selector's RPL != CPL
1638 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1642 if (!(seg_desc.type & 8))
1645 if (transfer == X86_TRANSFER_RET) {
1646 /* RET can never return to an inner privilege level. */
1649 /* Outer-privilege level return is not implemented */
1651 return X86EMUL_UNHANDLEABLE;
1653 if (transfer == X86_TRANSFER_RET || transfer == X86_TRANSFER_TASK_SWITCH) {
1654 if (seg_desc.type & 4) {
1663 } else { /* X86_TRANSFER_CALL_JMP */
1664 if (seg_desc.type & 4) {
1670 if (rpl > cpl || dpl != cpl)
1674 /* in long-mode d/b must be clear if l is set */
1675 if (seg_desc.d && seg_desc.l) {
1678 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1679 if (efer & EFER_LMA)
1683 /* CS(RPL) <- CPL */
1684 selector = (selector & 0xfffc) | cpl;
1687 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1690 case VCPU_SREG_LDTR:
1691 if (seg_desc.s || seg_desc.type != 2)
1694 default: /* DS, ES, FS, or GS */
1696 * segment is not a data or readable code segment or
1697 * ((segment is a data or nonconforming code segment)
1698 * and (both RPL and CPL > DPL))
1700 if ((seg_desc.type & 0xa) == 0x8 ||
1701 (((seg_desc.type & 0xc) != 0xc) &&
1702 (rpl > dpl && cpl > dpl)))
1708 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1713 /* mark segment as accessed */
1714 if (!(seg_desc.type & 1)) {
1716 ret = write_segment_descriptor(ctxt, selector,
1718 if (ret != X86EMUL_CONTINUE)
1721 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1722 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1723 if (ret != X86EMUL_CONTINUE)
1725 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1726 ((u64)base3 << 32), ctxt))
1727 return emulate_gp(ctxt, err_code);
1730 if (seg == VCPU_SREG_TR) {
1731 old_desc = seg_desc;
1732 seg_desc.type |= 2; /* busy */
1733 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1734 sizeof(seg_desc), &ctxt->exception);
1735 if (ret != X86EMUL_CONTINUE)
1739 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1742 return X86EMUL_CONTINUE;
1744 return emulate_exception(ctxt, err_vec, err_code, true);
1747 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1748 u16 selector, int seg)
1750 u8 cpl = ctxt->ops->cpl(ctxt);
1753 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1754 * they can load it at CPL<3 (Intel's manual says only LSS can,
1757 * However, the Intel manual says that putting IST=1/DPL=3 in
1758 * an interrupt gate will result in SS=3 (the AMD manual instead
1759 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1760 * and only forbid it here.
1762 if (seg == VCPU_SREG_SS && selector == 3 &&
1763 ctxt->mode == X86EMUL_MODE_PROT64)
1764 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1766 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1767 X86_TRANSFER_NONE, NULL);
1770 static void write_register_operand(struct operand *op)
1772 return assign_register(op->addr.reg, op->val, op->bytes);
1775 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1779 write_register_operand(op);
1782 if (ctxt->lock_prefix)
1783 return segmented_cmpxchg(ctxt,
1789 return segmented_write(ctxt,
1795 return segmented_write(ctxt,
1798 op->bytes * op->count);
1801 kvm_write_sse_reg(op->addr.xmm, &op->vec_val);
1804 kvm_write_mmx_reg(op->addr.mm, &op->mm_val);
1812 return X86EMUL_CONTINUE;
1815 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1817 struct segmented_address addr;
1819 rsp_increment(ctxt, -bytes);
1820 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1821 addr.seg = VCPU_SREG_SS;
1823 return segmented_write(ctxt, addr, data, bytes);
1826 static int em_push(struct x86_emulate_ctxt *ctxt)
1828 /* Disable writeback. */
1829 ctxt->dst.type = OP_NONE;
1830 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1833 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1834 void *dest, int len)
1837 struct segmented_address addr;
1839 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1840 addr.seg = VCPU_SREG_SS;
1841 rc = segmented_read(ctxt, addr, dest, len);
1842 if (rc != X86EMUL_CONTINUE)
1845 rsp_increment(ctxt, len);
1849 static int em_pop(struct x86_emulate_ctxt *ctxt)
1851 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1854 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1855 void *dest, int len)
1858 unsigned long val, change_mask;
1859 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1860 int cpl = ctxt->ops->cpl(ctxt);
1862 rc = emulate_pop(ctxt, &val, len);
1863 if (rc != X86EMUL_CONTINUE)
1866 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1867 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1868 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1869 X86_EFLAGS_AC | X86_EFLAGS_ID;
1871 switch(ctxt->mode) {
1872 case X86EMUL_MODE_PROT64:
1873 case X86EMUL_MODE_PROT32:
1874 case X86EMUL_MODE_PROT16:
1876 change_mask |= X86_EFLAGS_IOPL;
1878 change_mask |= X86_EFLAGS_IF;
1880 case X86EMUL_MODE_VM86:
1882 return emulate_gp(ctxt, 0);
1883 change_mask |= X86_EFLAGS_IF;
1885 default: /* real mode */
1886 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1890 *(unsigned long *)dest =
1891 (ctxt->eflags & ~change_mask) | (val & change_mask);
1896 static int em_popf(struct x86_emulate_ctxt *ctxt)
1898 ctxt->dst.type = OP_REG;
1899 ctxt->dst.addr.reg = &ctxt->eflags;
1900 ctxt->dst.bytes = ctxt->op_bytes;
1901 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1904 static int em_enter(struct x86_emulate_ctxt *ctxt)
1907 unsigned frame_size = ctxt->src.val;
1908 unsigned nesting_level = ctxt->src2.val & 31;
1912 return X86EMUL_UNHANDLEABLE;
1914 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1915 rc = push(ctxt, &rbp, stack_size(ctxt));
1916 if (rc != X86EMUL_CONTINUE)
1918 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1920 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1921 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1923 return X86EMUL_CONTINUE;
1926 static int em_leave(struct x86_emulate_ctxt *ctxt)
1928 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1930 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1933 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1935 int seg = ctxt->src2.val;
1937 ctxt->src.val = get_segment_selector(ctxt, seg);
1938 if (ctxt->op_bytes == 4) {
1939 rsp_increment(ctxt, -2);
1943 return em_push(ctxt);
1946 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1948 int seg = ctxt->src2.val;
1949 unsigned long selector;
1952 rc = emulate_pop(ctxt, &selector, 2);
1953 if (rc != X86EMUL_CONTINUE)
1956 if (ctxt->modrm_reg == VCPU_SREG_SS)
1957 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1958 if (ctxt->op_bytes > 2)
1959 rsp_increment(ctxt, ctxt->op_bytes - 2);
1961 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1965 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1967 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1968 int rc = X86EMUL_CONTINUE;
1969 int reg = VCPU_REGS_RAX;
1971 while (reg <= VCPU_REGS_RDI) {
1972 (reg == VCPU_REGS_RSP) ?
1973 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1976 if (rc != X86EMUL_CONTINUE)
1985 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1987 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1988 return em_push(ctxt);
1991 static int em_popa(struct x86_emulate_ctxt *ctxt)
1993 int rc = X86EMUL_CONTINUE;
1994 int reg = VCPU_REGS_RDI;
1997 while (reg >= VCPU_REGS_RAX) {
1998 if (reg == VCPU_REGS_RSP) {
1999 rsp_increment(ctxt, ctxt->op_bytes);
2003 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2004 if (rc != X86EMUL_CONTINUE)
2006 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2012 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2014 const struct x86_emulate_ops *ops = ctxt->ops;
2021 /* TODO: Add limit checks */
2022 ctxt->src.val = ctxt->eflags;
2024 if (rc != X86EMUL_CONTINUE)
2027 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2029 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2031 if (rc != X86EMUL_CONTINUE)
2034 ctxt->src.val = ctxt->_eip;
2036 if (rc != X86EMUL_CONTINUE)
2039 ops->get_idt(ctxt, &dt);
2041 eip_addr = dt.address + (irq << 2);
2042 cs_addr = dt.address + (irq << 2) + 2;
2044 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2045 if (rc != X86EMUL_CONTINUE)
2048 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2049 if (rc != X86EMUL_CONTINUE)
2052 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2053 if (rc != X86EMUL_CONTINUE)
2061 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2065 invalidate_registers(ctxt);
2066 rc = __emulate_int_real(ctxt, irq);
2067 if (rc == X86EMUL_CONTINUE)
2068 writeback_registers(ctxt);
2072 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2074 switch(ctxt->mode) {
2075 case X86EMUL_MODE_REAL:
2076 return __emulate_int_real(ctxt, irq);
2077 case X86EMUL_MODE_VM86:
2078 case X86EMUL_MODE_PROT16:
2079 case X86EMUL_MODE_PROT32:
2080 case X86EMUL_MODE_PROT64:
2082 /* Protected mode interrupts unimplemented yet */
2083 return X86EMUL_UNHANDLEABLE;
2087 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2089 int rc = X86EMUL_CONTINUE;
2090 unsigned long temp_eip = 0;
2091 unsigned long temp_eflags = 0;
2092 unsigned long cs = 0;
2093 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2094 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2095 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2096 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2097 X86_EFLAGS_AC | X86_EFLAGS_ID |
2099 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2102 /* TODO: Add stack limit check */
2104 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2106 if (rc != X86EMUL_CONTINUE)
2109 if (temp_eip & ~0xffff)
2110 return emulate_gp(ctxt, 0);
2112 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2114 if (rc != X86EMUL_CONTINUE)
2117 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2119 if (rc != X86EMUL_CONTINUE)
2122 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2124 if (rc != X86EMUL_CONTINUE)
2127 ctxt->_eip = temp_eip;
2129 if (ctxt->op_bytes == 4)
2130 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2131 else if (ctxt->op_bytes == 2) {
2132 ctxt->eflags &= ~0xffff;
2133 ctxt->eflags |= temp_eflags;
2136 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2137 ctxt->eflags |= X86_EFLAGS_FIXED;
2138 ctxt->ops->set_nmi_mask(ctxt, false);
2143 static int em_iret(struct x86_emulate_ctxt *ctxt)
2145 switch(ctxt->mode) {
2146 case X86EMUL_MODE_REAL:
2147 return emulate_iret_real(ctxt);
2148 case X86EMUL_MODE_VM86:
2149 case X86EMUL_MODE_PROT16:
2150 case X86EMUL_MODE_PROT32:
2151 case X86EMUL_MODE_PROT64:
2153 /* iret from protected mode unimplemented yet */
2154 return X86EMUL_UNHANDLEABLE;
2158 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2162 struct desc_struct new_desc;
2163 u8 cpl = ctxt->ops->cpl(ctxt);
2165 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2167 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2168 X86_TRANSFER_CALL_JMP,
2170 if (rc != X86EMUL_CONTINUE)
2173 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2174 /* Error handling is not implemented. */
2175 if (rc != X86EMUL_CONTINUE)
2176 return X86EMUL_UNHANDLEABLE;
2181 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2183 return assign_eip_near(ctxt, ctxt->src.val);
2186 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2191 old_eip = ctxt->_eip;
2192 rc = assign_eip_near(ctxt, ctxt->src.val);
2193 if (rc != X86EMUL_CONTINUE)
2195 ctxt->src.val = old_eip;
2200 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2202 u64 old = ctxt->dst.orig_val64;
2204 if (ctxt->dst.bytes == 16)
2205 return X86EMUL_UNHANDLEABLE;
2207 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2208 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2209 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2210 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2211 ctxt->eflags &= ~X86_EFLAGS_ZF;
2213 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2214 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2216 ctxt->eflags |= X86_EFLAGS_ZF;
2218 return X86EMUL_CONTINUE;
2221 static int em_ret(struct x86_emulate_ctxt *ctxt)
2226 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2227 if (rc != X86EMUL_CONTINUE)
2230 return assign_eip_near(ctxt, eip);
2233 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2236 unsigned long eip, cs;
2237 int cpl = ctxt->ops->cpl(ctxt);
2238 struct desc_struct new_desc;
2240 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2241 if (rc != X86EMUL_CONTINUE)
2243 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2244 if (rc != X86EMUL_CONTINUE)
2246 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2249 if (rc != X86EMUL_CONTINUE)
2251 rc = assign_eip_far(ctxt, eip, &new_desc);
2252 /* Error handling is not implemented. */
2253 if (rc != X86EMUL_CONTINUE)
2254 return X86EMUL_UNHANDLEABLE;
2259 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2263 rc = em_ret_far(ctxt);
2264 if (rc != X86EMUL_CONTINUE)
2266 rsp_increment(ctxt, ctxt->src.val);
2267 return X86EMUL_CONTINUE;
2270 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2272 /* Save real source value, then compare EAX against destination. */
2273 ctxt->dst.orig_val = ctxt->dst.val;
2274 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2275 ctxt->src.orig_val = ctxt->src.val;
2276 ctxt->src.val = ctxt->dst.orig_val;
2277 fastop(ctxt, em_cmp);
2279 if (ctxt->eflags & X86_EFLAGS_ZF) {
2280 /* Success: write back to memory; no update of EAX */
2281 ctxt->src.type = OP_NONE;
2282 ctxt->dst.val = ctxt->src.orig_val;
2284 /* Failure: write the value we saw to EAX. */
2285 ctxt->src.type = OP_REG;
2286 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2287 ctxt->src.val = ctxt->dst.orig_val;
2288 /* Create write-cycle to dest by writing the same value */
2289 ctxt->dst.val = ctxt->dst.orig_val;
2291 return X86EMUL_CONTINUE;
2294 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2296 int seg = ctxt->src2.val;
2300 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2302 rc = load_segment_descriptor(ctxt, sel, seg);
2303 if (rc != X86EMUL_CONTINUE)
2306 ctxt->dst.val = ctxt->src.val;
2310 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2312 #ifdef CONFIG_X86_64
2313 return ctxt->ops->guest_has_long_mode(ctxt);
2319 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2321 desc->g = (flags >> 23) & 1;
2322 desc->d = (flags >> 22) & 1;
2323 desc->l = (flags >> 21) & 1;
2324 desc->avl = (flags >> 20) & 1;
2325 desc->p = (flags >> 15) & 1;
2326 desc->dpl = (flags >> 13) & 3;
2327 desc->s = (flags >> 12) & 1;
2328 desc->type = (flags >> 8) & 15;
2331 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2334 struct desc_struct desc;
2338 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2341 offset = 0x7f84 + n * 12;
2343 offset = 0x7f2c + (n - 3) * 12;
2345 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2346 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2347 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2348 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2349 return X86EMUL_CONTINUE;
2352 #ifdef CONFIG_X86_64
2353 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2356 struct desc_struct desc;
2361 offset = 0x7e00 + n * 16;
2363 selector = GET_SMSTATE(u16, smstate, offset);
2364 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2365 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2366 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2367 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2369 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2370 return X86EMUL_CONTINUE;
2374 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2375 u64 cr0, u64 cr3, u64 cr4)
2380 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2382 if (cr4 & X86_CR4_PCIDE) {
2387 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2389 return X86EMUL_UNHANDLEABLE;
2392 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2393 * Then enable protected mode. However, PCID cannot be enabled
2394 * if EFER.LMA=0, so set it separately.
2396 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2398 return X86EMUL_UNHANDLEABLE;
2400 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2402 return X86EMUL_UNHANDLEABLE;
2404 if (cr4 & X86_CR4_PCIDE) {
2405 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2407 return X86EMUL_UNHANDLEABLE;
2409 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2411 return X86EMUL_UNHANDLEABLE;
2416 return X86EMUL_CONTINUE;
2419 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2420 const char *smstate)
2422 struct desc_struct desc;
2425 u32 val, cr0, cr3, cr4;
2428 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2429 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2430 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2431 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2433 for (i = 0; i < NR_EMULATOR_GPRS; i++)
2434 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2436 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2438 if (ctxt->ops->set_dr(ctxt, 6, val))
2439 return X86EMUL_UNHANDLEABLE;
2441 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2443 if (ctxt->ops->set_dr(ctxt, 7, val))
2444 return X86EMUL_UNHANDLEABLE;
2446 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2447 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2448 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2449 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2450 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2452 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2453 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2454 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2455 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2456 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2458 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2459 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2460 ctxt->ops->set_gdt(ctxt, &dt);
2462 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2463 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2464 ctxt->ops->set_idt(ctxt, &dt);
2466 for (i = 0; i < 6; i++) {
2467 int r = rsm_load_seg_32(ctxt, smstate, i);
2468 if (r != X86EMUL_CONTINUE)
2472 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2474 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2476 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2479 #ifdef CONFIG_X86_64
2480 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2481 const char *smstate)
2483 struct desc_struct desc;
2485 u64 val, cr0, cr3, cr4;
2490 for (i = 0; i < NR_EMULATOR_GPRS; i++)
2491 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2493 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2494 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2496 val = GET_SMSTATE(u64, smstate, 0x7f68);
2498 if (ctxt->ops->set_dr(ctxt, 6, val))
2499 return X86EMUL_UNHANDLEABLE;
2501 val = GET_SMSTATE(u64, smstate, 0x7f60);
2503 if (ctxt->ops->set_dr(ctxt, 7, val))
2504 return X86EMUL_UNHANDLEABLE;
2506 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2507 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2508 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2509 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2510 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2512 if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))
2513 return X86EMUL_UNHANDLEABLE;
2515 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2516 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2517 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2518 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2519 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2520 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2522 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2523 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2524 ctxt->ops->set_idt(ctxt, &dt);
2526 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2527 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2528 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2529 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2530 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2531 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2533 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2534 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2535 ctxt->ops->set_gdt(ctxt, &dt);
2537 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2538 if (r != X86EMUL_CONTINUE)
2541 for (i = 0; i < 6; i++) {
2542 r = rsm_load_seg_64(ctxt, smstate, i);
2543 if (r != X86EMUL_CONTINUE)
2547 return X86EMUL_CONTINUE;
2551 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2553 unsigned long cr0, cr4, efer;
2558 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2559 return emulate_ud(ctxt);
2561 smbase = ctxt->ops->get_smbase(ctxt);
2563 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2564 if (ret != X86EMUL_CONTINUE)
2565 return X86EMUL_UNHANDLEABLE;
2567 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2568 ctxt->ops->set_nmi_mask(ctxt, false);
2570 ctxt->ops->exiting_smm(ctxt);
2573 * Get back to real mode, to prepare a safe state in which to load
2574 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2575 * supports long mode.
2577 if (emulator_has_longmode(ctxt)) {
2578 struct desc_struct cs_desc;
2580 /* Zero CR4.PCIDE before CR0.PG. */
2581 cr4 = ctxt->ops->get_cr(ctxt, 4);
2582 if (cr4 & X86_CR4_PCIDE)
2583 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2585 /* A 32-bit code segment is required to clear EFER.LMA. */
2586 memset(&cs_desc, 0, sizeof(cs_desc));
2588 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2589 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2592 /* For the 64-bit case, this will clear EFER.LMA. */
2593 cr0 = ctxt->ops->get_cr(ctxt, 0);
2594 if (cr0 & X86_CR0_PE)
2595 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2597 if (emulator_has_longmode(ctxt)) {
2598 /* Clear CR4.PAE before clearing EFER.LME. */
2599 cr4 = ctxt->ops->get_cr(ctxt, 4);
2600 if (cr4 & X86_CR4_PAE)
2601 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2603 /* And finally go back to 32-bit mode. */
2605 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2609 * Give leave_smm() a chance to make ISA-specific changes to the vCPU
2610 * state (e.g. enter guest mode) before loading state from the SMM
2613 if (ctxt->ops->leave_smm(ctxt, buf))
2614 goto emulate_shutdown;
2616 #ifdef CONFIG_X86_64
2617 if (emulator_has_longmode(ctxt))
2618 ret = rsm_load_state_64(ctxt, buf);
2621 ret = rsm_load_state_32(ctxt, buf);
2623 if (ret != X86EMUL_CONTINUE)
2624 goto emulate_shutdown;
2627 * Note, the ctxt->ops callbacks are responsible for handling side
2628 * effects when writing MSRs and CRs, e.g. MMU context resets, CPUID
2629 * runtime updates, etc... If that changes, e.g. this flow is moved
2630 * out of the emulator to make it look more like enter_smm(), then
2631 * those side effects need to be explicitly handled for both success
2634 return X86EMUL_CONTINUE;
2637 ctxt->ops->triple_fault(ctxt);
2638 return X86EMUL_CONTINUE;
2642 setup_syscalls_segments(struct desc_struct *cs, struct desc_struct *ss)
2644 cs->l = 0; /* will be adjusted later */
2645 set_desc_base(cs, 0); /* flat segment */
2646 cs->g = 1; /* 4kb granularity */
2647 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2648 cs->type = 0x0b; /* Read, Execute, Accessed */
2650 cs->dpl = 0; /* will be adjusted later */
2655 set_desc_base(ss, 0); /* flat segment */
2656 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2657 ss->g = 1; /* 4kb granularity */
2659 ss->type = 0x03; /* Read/Write, Accessed */
2660 ss->d = 1; /* 32bit stack segment */
2667 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2669 u32 eax, ebx, ecx, edx;
2672 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2673 return is_guest_vendor_intel(ebx, ecx, edx);
2676 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2678 const struct x86_emulate_ops *ops = ctxt->ops;
2679 u32 eax, ebx, ecx, edx;
2682 * syscall should always be enabled in longmode - so only become
2683 * vendor specific (cpuid) if other modes are active...
2685 if (ctxt->mode == X86EMUL_MODE_PROT64)
2690 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2692 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
2693 * 64bit guest with a 32bit compat-app running will #UD !! While this
2694 * behaviour can be fixed (by emulating) into AMD response - CPUs of
2695 * AMD can't behave like Intel.
2697 if (is_guest_vendor_intel(ebx, ecx, edx))
2700 if (is_guest_vendor_amd(ebx, ecx, edx) ||
2701 is_guest_vendor_hygon(ebx, ecx, edx))
2705 * default: (not Intel, not AMD, not Hygon), apply Intel's
2711 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2713 const struct x86_emulate_ops *ops = ctxt->ops;
2714 struct desc_struct cs, ss;
2719 /* syscall is not available in real mode */
2720 if (ctxt->mode == X86EMUL_MODE_REAL ||
2721 ctxt->mode == X86EMUL_MODE_VM86)
2722 return emulate_ud(ctxt);
2724 if (!(em_syscall_is_enabled(ctxt)))
2725 return emulate_ud(ctxt);
2727 ops->get_msr(ctxt, MSR_EFER, &efer);
2728 if (!(efer & EFER_SCE))
2729 return emulate_ud(ctxt);
2731 setup_syscalls_segments(&cs, &ss);
2732 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2734 cs_sel = (u16)(msr_data & 0xfffc);
2735 ss_sel = (u16)(msr_data + 8);
2737 if (efer & EFER_LMA) {
2741 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2742 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2744 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2745 if (efer & EFER_LMA) {
2746 #ifdef CONFIG_X86_64
2747 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2750 ctxt->mode == X86EMUL_MODE_PROT64 ?
2751 MSR_LSTAR : MSR_CSTAR, &msr_data);
2752 ctxt->_eip = msr_data;
2754 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2755 ctxt->eflags &= ~msr_data;
2756 ctxt->eflags |= X86_EFLAGS_FIXED;
2760 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2761 ctxt->_eip = (u32)msr_data;
2763 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2766 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2767 return X86EMUL_CONTINUE;
2770 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2772 const struct x86_emulate_ops *ops = ctxt->ops;
2773 struct desc_struct cs, ss;
2778 ops->get_msr(ctxt, MSR_EFER, &efer);
2779 /* inject #GP if in real mode */
2780 if (ctxt->mode == X86EMUL_MODE_REAL)
2781 return emulate_gp(ctxt, 0);
2784 * Not recognized on AMD in compat mode (but is recognized in legacy
2787 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2788 && !vendor_intel(ctxt))
2789 return emulate_ud(ctxt);
2791 /* sysenter/sysexit have not been tested in 64bit mode. */
2792 if (ctxt->mode == X86EMUL_MODE_PROT64)
2793 return X86EMUL_UNHANDLEABLE;
2795 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2796 if ((msr_data & 0xfffc) == 0x0)
2797 return emulate_gp(ctxt, 0);
2799 setup_syscalls_segments(&cs, &ss);
2800 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2801 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2802 ss_sel = cs_sel + 8;
2803 if (efer & EFER_LMA) {
2808 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2809 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2811 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2812 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2814 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2815 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2817 if (efer & EFER_LMA)
2818 ctxt->mode = X86EMUL_MODE_PROT64;
2820 return X86EMUL_CONTINUE;
2823 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2825 const struct x86_emulate_ops *ops = ctxt->ops;
2826 struct desc_struct cs, ss;
2827 u64 msr_data, rcx, rdx;
2829 u16 cs_sel = 0, ss_sel = 0;
2831 /* inject #GP if in real mode or Virtual 8086 mode */
2832 if (ctxt->mode == X86EMUL_MODE_REAL ||
2833 ctxt->mode == X86EMUL_MODE_VM86)
2834 return emulate_gp(ctxt, 0);
2836 setup_syscalls_segments(&cs, &ss);
2838 if ((ctxt->rex_prefix & 0x8) != 0x0)
2839 usermode = X86EMUL_MODE_PROT64;
2841 usermode = X86EMUL_MODE_PROT32;
2843 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2844 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2848 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2850 case X86EMUL_MODE_PROT32:
2851 cs_sel = (u16)(msr_data + 16);
2852 if ((msr_data & 0xfffc) == 0x0)
2853 return emulate_gp(ctxt, 0);
2854 ss_sel = (u16)(msr_data + 24);
2858 case X86EMUL_MODE_PROT64:
2859 cs_sel = (u16)(msr_data + 32);
2860 if (msr_data == 0x0)
2861 return emulate_gp(ctxt, 0);
2862 ss_sel = cs_sel + 8;
2865 if (emul_is_noncanonical_address(rcx, ctxt) ||
2866 emul_is_noncanonical_address(rdx, ctxt))
2867 return emulate_gp(ctxt, 0);
2870 cs_sel |= SEGMENT_RPL_MASK;
2871 ss_sel |= SEGMENT_RPL_MASK;
2873 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2874 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2877 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2879 return X86EMUL_CONTINUE;
2882 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2885 if (ctxt->mode == X86EMUL_MODE_REAL)
2887 if (ctxt->mode == X86EMUL_MODE_VM86)
2889 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2890 return ctxt->ops->cpl(ctxt) > iopl;
2893 #define VMWARE_PORT_VMPORT (0x5658)
2894 #define VMWARE_PORT_VMRPC (0x5659)
2896 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2899 const struct x86_emulate_ops *ops = ctxt->ops;
2900 struct desc_struct tr_seg;
2903 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2904 unsigned mask = (1 << len) - 1;
2908 * VMware allows access to these ports even if denied
2909 * by TSS I/O permission bitmap. Mimic behavior.
2911 if (enable_vmware_backdoor &&
2912 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2915 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2918 if (desc_limit_scaled(&tr_seg) < 103)
2920 base = get_desc_base(&tr_seg);
2921 #ifdef CONFIG_X86_64
2922 base |= ((u64)base3) << 32;
2924 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2925 if (r != X86EMUL_CONTINUE)
2927 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2929 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2930 if (r != X86EMUL_CONTINUE)
2932 if ((perm >> bit_idx) & mask)
2937 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2943 if (emulator_bad_iopl(ctxt))
2944 if (!emulator_io_port_access_allowed(ctxt, port, len))
2947 ctxt->perm_ok = true;
2952 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2955 * Intel CPUs mask the counter and pointers in quite strange
2956 * manner when ECX is zero due to REP-string optimizations.
2958 #ifdef CONFIG_X86_64
2959 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2962 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2965 case 0xa4: /* movsb */
2966 case 0xa5: /* movsd/w */
2967 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2969 case 0xaa: /* stosb */
2970 case 0xab: /* stosd/w */
2971 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2976 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2977 struct tss_segment_16 *tss)
2979 tss->ip = ctxt->_eip;
2980 tss->flag = ctxt->eflags;
2981 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2982 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2983 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2984 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2985 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2986 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2987 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2988 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2990 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2991 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2992 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2993 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2994 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2997 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2998 struct tss_segment_16 *tss)
3003 ctxt->_eip = tss->ip;
3004 ctxt->eflags = tss->flag | 2;
3005 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3006 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3007 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3008 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3009 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3010 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3011 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3012 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3015 * SDM says that segment selectors are loaded before segment
3018 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3019 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3020 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3021 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3022 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3027 * Now load segment descriptors. If fault happens at this stage
3028 * it is handled in a context of new task
3030 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3031 X86_TRANSFER_TASK_SWITCH, NULL);
3032 if (ret != X86EMUL_CONTINUE)
3034 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3035 X86_TRANSFER_TASK_SWITCH, NULL);
3036 if (ret != X86EMUL_CONTINUE)
3038 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3039 X86_TRANSFER_TASK_SWITCH, NULL);
3040 if (ret != X86EMUL_CONTINUE)
3042 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3043 X86_TRANSFER_TASK_SWITCH, NULL);
3044 if (ret != X86EMUL_CONTINUE)
3046 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3047 X86_TRANSFER_TASK_SWITCH, NULL);
3048 if (ret != X86EMUL_CONTINUE)
3051 return X86EMUL_CONTINUE;
3054 static int task_switch_16(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3055 ulong old_tss_base, struct desc_struct *new_desc)
3057 struct tss_segment_16 tss_seg;
3059 u32 new_tss_base = get_desc_base(new_desc);
3061 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3062 if (ret != X86EMUL_CONTINUE)
3065 save_state_to_tss16(ctxt, &tss_seg);
3067 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3068 if (ret != X86EMUL_CONTINUE)
3071 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3072 if (ret != X86EMUL_CONTINUE)
3075 if (old_tss_sel != 0xffff) {
3076 tss_seg.prev_task_link = old_tss_sel;
3078 ret = linear_write_system(ctxt, new_tss_base,
3079 &tss_seg.prev_task_link,
3080 sizeof(tss_seg.prev_task_link));
3081 if (ret != X86EMUL_CONTINUE)
3085 return load_state_from_tss16(ctxt, &tss_seg);
3088 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3089 struct tss_segment_32 *tss)
3091 /* CR3 and ldt selector are not saved intentionally */
3092 tss->eip = ctxt->_eip;
3093 tss->eflags = ctxt->eflags;
3094 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3095 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3096 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3097 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3098 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3099 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3100 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3101 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3103 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3104 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3105 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3106 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3107 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3108 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3111 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3112 struct tss_segment_32 *tss)
3117 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3118 return emulate_gp(ctxt, 0);
3119 ctxt->_eip = tss->eip;
3120 ctxt->eflags = tss->eflags | 2;
3122 /* General purpose registers */
3123 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3124 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3125 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3126 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3127 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3128 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3129 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3130 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3133 * SDM says that segment selectors are loaded before segment
3134 * descriptors. This is important because CPL checks will
3137 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3138 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3139 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3140 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3141 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3142 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3143 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3146 * If we're switching between Protected Mode and VM86, we need to make
3147 * sure to update the mode before loading the segment descriptors so
3148 * that the selectors are interpreted correctly.
3150 if (ctxt->eflags & X86_EFLAGS_VM) {
3151 ctxt->mode = X86EMUL_MODE_VM86;
3154 ctxt->mode = X86EMUL_MODE_PROT32;
3159 * Now load segment descriptors. If fault happens at this stage
3160 * it is handled in a context of new task
3162 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3163 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3164 if (ret != X86EMUL_CONTINUE)
3166 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3167 X86_TRANSFER_TASK_SWITCH, NULL);
3168 if (ret != X86EMUL_CONTINUE)
3170 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3171 X86_TRANSFER_TASK_SWITCH, NULL);
3172 if (ret != X86EMUL_CONTINUE)
3174 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3175 X86_TRANSFER_TASK_SWITCH, NULL);
3176 if (ret != X86EMUL_CONTINUE)
3178 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3179 X86_TRANSFER_TASK_SWITCH, NULL);
3180 if (ret != X86EMUL_CONTINUE)
3182 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3183 X86_TRANSFER_TASK_SWITCH, NULL);
3184 if (ret != X86EMUL_CONTINUE)
3186 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3187 X86_TRANSFER_TASK_SWITCH, NULL);
3192 static int task_switch_32(struct x86_emulate_ctxt *ctxt, u16 old_tss_sel,
3193 ulong old_tss_base, struct desc_struct *new_desc)
3195 struct tss_segment_32 tss_seg;
3197 u32 new_tss_base = get_desc_base(new_desc);
3198 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3199 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3201 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3202 if (ret != X86EMUL_CONTINUE)
3205 save_state_to_tss32(ctxt, &tss_seg);
3207 /* Only GP registers and segment selectors are saved */
3208 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3209 ldt_sel_offset - eip_offset);
3210 if (ret != X86EMUL_CONTINUE)
3213 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3214 if (ret != X86EMUL_CONTINUE)
3217 if (old_tss_sel != 0xffff) {
3218 tss_seg.prev_task_link = old_tss_sel;
3220 ret = linear_write_system(ctxt, new_tss_base,
3221 &tss_seg.prev_task_link,
3222 sizeof(tss_seg.prev_task_link));
3223 if (ret != X86EMUL_CONTINUE)
3227 return load_state_from_tss32(ctxt, &tss_seg);
3230 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3231 u16 tss_selector, int idt_index, int reason,
3232 bool has_error_code, u32 error_code)
3234 const struct x86_emulate_ops *ops = ctxt->ops;
3235 struct desc_struct curr_tss_desc, next_tss_desc;
3237 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3238 ulong old_tss_base =
3239 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3241 ulong desc_addr, dr7;
3243 /* FIXME: old_tss_base == ~0 ? */
3245 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3246 if (ret != X86EMUL_CONTINUE)
3248 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3249 if (ret != X86EMUL_CONTINUE)
3252 /* FIXME: check that next_tss_desc is tss */
3255 * Check privileges. The three cases are task switch caused by...
3257 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3258 * 2. Exception/IRQ/iret: No check is performed
3259 * 3. jmp/call to TSS/task-gate: No check is performed since the
3260 * hardware checks it before exiting.
3262 if (reason == TASK_SWITCH_GATE) {
3263 if (idt_index != -1) {
3264 /* Software interrupts */
3265 struct desc_struct task_gate_desc;
3268 ret = read_interrupt_descriptor(ctxt, idt_index,
3270 if (ret != X86EMUL_CONTINUE)
3273 dpl = task_gate_desc.dpl;
3274 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3275 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3279 desc_limit = desc_limit_scaled(&next_tss_desc);
3280 if (!next_tss_desc.p ||
3281 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3282 desc_limit < 0x2b)) {
3283 return emulate_ts(ctxt, tss_selector & 0xfffc);
3286 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3287 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3288 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3291 if (reason == TASK_SWITCH_IRET)
3292 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3294 /* set back link to prev task only if NT bit is set in eflags
3295 note that old_tss_sel is not used after this point */
3296 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3297 old_tss_sel = 0xffff;
3299 if (next_tss_desc.type & 8)
3300 ret = task_switch_32(ctxt, old_tss_sel, old_tss_base, &next_tss_desc);
3302 ret = task_switch_16(ctxt, old_tss_sel,
3303 old_tss_base, &next_tss_desc);
3304 if (ret != X86EMUL_CONTINUE)
3307 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3308 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3310 if (reason != TASK_SWITCH_IRET) {
3311 next_tss_desc.type |= (1 << 1); /* set busy flag */
3312 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3315 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3316 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3318 if (has_error_code) {
3319 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3320 ctxt->lock_prefix = 0;
3321 ctxt->src.val = (unsigned long) error_code;
3322 ret = em_push(ctxt);
3325 ops->get_dr(ctxt, 7, &dr7);
3326 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3331 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3332 u16 tss_selector, int idt_index, int reason,
3333 bool has_error_code, u32 error_code)
3337 invalidate_registers(ctxt);
3338 ctxt->_eip = ctxt->eip;
3339 ctxt->dst.type = OP_NONE;
3341 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3342 has_error_code, error_code);
3344 if (rc == X86EMUL_CONTINUE) {
3345 ctxt->eip = ctxt->_eip;
3346 writeback_registers(ctxt);
3349 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3352 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3355 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3357 register_address_increment(ctxt, reg, df * op->bytes);
3358 op->addr.mem.ea = register_address(ctxt, reg);
3361 static int em_das(struct x86_emulate_ctxt *ctxt)
3364 bool af, cf, old_cf;
3366 cf = ctxt->eflags & X86_EFLAGS_CF;
3372 af = ctxt->eflags & X86_EFLAGS_AF;
3373 if ((al & 0x0f) > 9 || af) {
3375 cf = old_cf | (al >= 250);
3380 if (old_al > 0x99 || old_cf) {
3386 /* Set PF, ZF, SF */
3387 ctxt->src.type = OP_IMM;
3389 ctxt->src.bytes = 1;
3390 fastop(ctxt, em_or);
3391 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3393 ctxt->eflags |= X86_EFLAGS_CF;
3395 ctxt->eflags |= X86_EFLAGS_AF;
3396 return X86EMUL_CONTINUE;
3399 static int em_aam(struct x86_emulate_ctxt *ctxt)
3403 if (ctxt->src.val == 0)
3404 return emulate_de(ctxt);
3406 al = ctxt->dst.val & 0xff;
3407 ah = al / ctxt->src.val;
3408 al %= ctxt->src.val;
3410 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3412 /* Set PF, ZF, SF */
3413 ctxt->src.type = OP_IMM;
3415 ctxt->src.bytes = 1;
3416 fastop(ctxt, em_or);
3418 return X86EMUL_CONTINUE;
3421 static int em_aad(struct x86_emulate_ctxt *ctxt)
3423 u8 al = ctxt->dst.val & 0xff;
3424 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3426 al = (al + (ah * ctxt->src.val)) & 0xff;
3428 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3430 /* Set PF, ZF, SF */
3431 ctxt->src.type = OP_IMM;
3433 ctxt->src.bytes = 1;
3434 fastop(ctxt, em_or);
3436 return X86EMUL_CONTINUE;
3439 static int em_call(struct x86_emulate_ctxt *ctxt)
3442 long rel = ctxt->src.val;
3444 ctxt->src.val = (unsigned long)ctxt->_eip;
3445 rc = jmp_rel(ctxt, rel);
3446 if (rc != X86EMUL_CONTINUE)
3448 return em_push(ctxt);
3451 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3456 struct desc_struct old_desc, new_desc;
3457 const struct x86_emulate_ops *ops = ctxt->ops;
3458 int cpl = ctxt->ops->cpl(ctxt);
3459 enum x86emul_mode prev_mode = ctxt->mode;
3461 old_eip = ctxt->_eip;
3462 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3464 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3465 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3466 X86_TRANSFER_CALL_JMP, &new_desc);
3467 if (rc != X86EMUL_CONTINUE)
3470 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3471 if (rc != X86EMUL_CONTINUE)
3474 ctxt->src.val = old_cs;
3476 if (rc != X86EMUL_CONTINUE)
3479 ctxt->src.val = old_eip;
3481 /* If we failed, we tainted the memory, but the very least we should
3483 if (rc != X86EMUL_CONTINUE) {
3484 pr_warn_once("faulting far call emulation tainted memory\n");
3489 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3490 ctxt->mode = prev_mode;
3495 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3500 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3501 if (rc != X86EMUL_CONTINUE)
3503 rc = assign_eip_near(ctxt, eip);
3504 if (rc != X86EMUL_CONTINUE)
3506 rsp_increment(ctxt, ctxt->src.val);
3507 return X86EMUL_CONTINUE;
3510 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3512 /* Write back the register source. */
3513 ctxt->src.val = ctxt->dst.val;
3514 write_register_operand(&ctxt->src);
3516 /* Write back the memory destination with implicit LOCK prefix. */
3517 ctxt->dst.val = ctxt->src.orig_val;
3518 ctxt->lock_prefix = 1;
3519 return X86EMUL_CONTINUE;
3522 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3524 ctxt->dst.val = ctxt->src2.val;
3525 return fastop(ctxt, em_imul);
3528 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3530 ctxt->dst.type = OP_REG;
3531 ctxt->dst.bytes = ctxt->src.bytes;
3532 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3533 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3535 return X86EMUL_CONTINUE;
3538 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3542 if (!ctxt->ops->guest_has_rdpid(ctxt))
3543 return emulate_ud(ctxt);
3545 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);
3546 ctxt->dst.val = tsc_aux;
3547 return X86EMUL_CONTINUE;
3550 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3554 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3555 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3556 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3557 return X86EMUL_CONTINUE;
3560 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3564 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3565 return emulate_gp(ctxt, 0);
3566 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3567 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3568 return X86EMUL_CONTINUE;
3571 static int em_mov(struct x86_emulate_ctxt *ctxt)
3573 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3574 return X86EMUL_CONTINUE;
3577 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3581 if (!ctxt->ops->guest_has_movbe(ctxt))
3582 return emulate_ud(ctxt);
3584 switch (ctxt->op_bytes) {
3587 * From MOVBE definition: "...When the operand size is 16 bits,
3588 * the upper word of the destination register remains unchanged
3591 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3592 * rules so we have to do the operation almost per hand.
3594 tmp = (u16)ctxt->src.val;
3595 ctxt->dst.val &= ~0xffffUL;
3596 ctxt->dst.val |= (unsigned long)swab16(tmp);
3599 ctxt->dst.val = swab32((u32)ctxt->src.val);
3602 ctxt->dst.val = swab64(ctxt->src.val);
3607 return X86EMUL_CONTINUE;
3610 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3612 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3613 return emulate_gp(ctxt, 0);
3615 /* Disable writeback. */
3616 ctxt->dst.type = OP_NONE;
3617 return X86EMUL_CONTINUE;
3620 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3624 if (ctxt->mode == X86EMUL_MODE_PROT64)
3625 val = ctxt->src.val & ~0ULL;
3627 val = ctxt->src.val & ~0U;
3629 /* #UD condition is already handled. */
3630 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3631 return emulate_gp(ctxt, 0);
3633 /* Disable writeback. */
3634 ctxt->dst.type = OP_NONE;
3635 return X86EMUL_CONTINUE;
3638 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3640 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3644 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3645 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3646 r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data);
3648 if (r == X86EMUL_IO_NEEDED)
3652 return emulate_gp(ctxt, 0);
3654 return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
3657 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3659 u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3663 r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data);
3665 if (r == X86EMUL_IO_NEEDED)
3669 return emulate_gp(ctxt, 0);
3671 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3672 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3673 return X86EMUL_CONTINUE;
3676 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3678 if (segment > VCPU_SREG_GS &&
3679 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3680 ctxt->ops->cpl(ctxt) > 0)
3681 return emulate_gp(ctxt, 0);
3683 ctxt->dst.val = get_segment_selector(ctxt, segment);
3684 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3685 ctxt->dst.bytes = 2;
3686 return X86EMUL_CONTINUE;
3689 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3691 if (ctxt->modrm_reg > VCPU_SREG_GS)
3692 return emulate_ud(ctxt);
3694 return em_store_sreg(ctxt, ctxt->modrm_reg);
3697 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3699 u16 sel = ctxt->src.val;
3701 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3702 return emulate_ud(ctxt);
3704 if (ctxt->modrm_reg == VCPU_SREG_SS)
3705 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3707 /* Disable writeback. */
3708 ctxt->dst.type = OP_NONE;
3709 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3712 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3714 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3717 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3719 u16 sel = ctxt->src.val;
3721 /* Disable writeback. */
3722 ctxt->dst.type = OP_NONE;
3723 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3726 static int em_str(struct x86_emulate_ctxt *ctxt)
3728 return em_store_sreg(ctxt, VCPU_SREG_TR);
3731 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3733 u16 sel = ctxt->src.val;
3735 /* Disable writeback. */
3736 ctxt->dst.type = OP_NONE;
3737 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3740 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3745 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3746 if (rc == X86EMUL_CONTINUE)
3747 ctxt->ops->invlpg(ctxt, linear);
3748 /* Disable writeback. */
3749 ctxt->dst.type = OP_NONE;
3750 return X86EMUL_CONTINUE;
3753 static int em_clts(struct x86_emulate_ctxt *ctxt)
3757 cr0 = ctxt->ops->get_cr(ctxt, 0);
3759 ctxt->ops->set_cr(ctxt, 0, cr0);
3760 return X86EMUL_CONTINUE;
3763 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3765 int rc = ctxt->ops->fix_hypercall(ctxt);
3767 if (rc != X86EMUL_CONTINUE)
3770 /* Let the processor re-execute the fixed hypercall */
3771 ctxt->_eip = ctxt->eip;
3772 /* Disable writeback. */
3773 ctxt->dst.type = OP_NONE;
3774 return X86EMUL_CONTINUE;
3777 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3778 void (*get)(struct x86_emulate_ctxt *ctxt,
3779 struct desc_ptr *ptr))
3781 struct desc_ptr desc_ptr;
3783 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3784 ctxt->ops->cpl(ctxt) > 0)
3785 return emulate_gp(ctxt, 0);
3787 if (ctxt->mode == X86EMUL_MODE_PROT64)
3789 get(ctxt, &desc_ptr);
3790 if (ctxt->op_bytes == 2) {
3792 desc_ptr.address &= 0x00ffffff;
3794 /* Disable writeback. */
3795 ctxt->dst.type = OP_NONE;
3796 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3797 &desc_ptr, 2 + ctxt->op_bytes);
3800 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3802 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3805 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3807 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3810 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3812 struct desc_ptr desc_ptr;
3815 if (ctxt->mode == X86EMUL_MODE_PROT64)
3817 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3818 &desc_ptr.size, &desc_ptr.address,
3820 if (rc != X86EMUL_CONTINUE)
3822 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3823 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3824 return emulate_gp(ctxt, 0);
3826 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3828 ctxt->ops->set_idt(ctxt, &desc_ptr);
3829 /* Disable writeback. */
3830 ctxt->dst.type = OP_NONE;
3831 return X86EMUL_CONTINUE;
3834 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3836 return em_lgdt_lidt(ctxt, true);
3839 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3841 return em_lgdt_lidt(ctxt, false);
3844 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3846 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3847 ctxt->ops->cpl(ctxt) > 0)
3848 return emulate_gp(ctxt, 0);
3850 if (ctxt->dst.type == OP_MEM)
3851 ctxt->dst.bytes = 2;
3852 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3853 return X86EMUL_CONTINUE;
3856 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3858 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3859 | (ctxt->src.val & 0x0f));
3860 ctxt->dst.type = OP_NONE;
3861 return X86EMUL_CONTINUE;
3864 static int em_loop(struct x86_emulate_ctxt *ctxt)
3866 int rc = X86EMUL_CONTINUE;
3868 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3869 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3870 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3871 rc = jmp_rel(ctxt, ctxt->src.val);
3876 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3878 int rc = X86EMUL_CONTINUE;
3880 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3881 rc = jmp_rel(ctxt, ctxt->src.val);
3886 static int em_in(struct x86_emulate_ctxt *ctxt)
3888 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3890 return X86EMUL_IO_NEEDED;
3892 return X86EMUL_CONTINUE;
3895 static int em_out(struct x86_emulate_ctxt *ctxt)
3897 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3899 /* Disable writeback. */
3900 ctxt->dst.type = OP_NONE;
3901 return X86EMUL_CONTINUE;
3904 static int em_cli(struct x86_emulate_ctxt *ctxt)
3906 if (emulator_bad_iopl(ctxt))
3907 return emulate_gp(ctxt, 0);
3909 ctxt->eflags &= ~X86_EFLAGS_IF;
3910 return X86EMUL_CONTINUE;
3913 static int em_sti(struct x86_emulate_ctxt *ctxt)
3915 if (emulator_bad_iopl(ctxt))
3916 return emulate_gp(ctxt, 0);
3918 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3919 ctxt->eflags |= X86_EFLAGS_IF;
3920 return X86EMUL_CONTINUE;
3923 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3925 u32 eax, ebx, ecx, edx;
3928 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3929 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3930 ctxt->ops->cpl(ctxt)) {
3931 return emulate_gp(ctxt, 0);
3934 eax = reg_read(ctxt, VCPU_REGS_RAX);
3935 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3936 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3937 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3938 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3939 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3940 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3941 return X86EMUL_CONTINUE;
3944 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3948 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3950 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3952 ctxt->eflags &= ~0xffUL;
3953 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3954 return X86EMUL_CONTINUE;
3957 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3959 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3960 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3961 return X86EMUL_CONTINUE;
3964 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3966 switch (ctxt->op_bytes) {
3967 #ifdef CONFIG_X86_64
3969 asm("bswap %0" : "+r"(ctxt->dst.val));
3973 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3976 return X86EMUL_CONTINUE;
3979 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3981 /* emulating clflush regardless of cpuid */
3982 return X86EMUL_CONTINUE;
3985 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3987 /* emulating clflushopt regardless of cpuid */
3988 return X86EMUL_CONTINUE;
3991 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3993 ctxt->dst.val = (s32) ctxt->src.val;
3994 return X86EMUL_CONTINUE;
3997 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3999 if (!ctxt->ops->guest_has_fxsr(ctxt))
4000 return emulate_ud(ctxt);
4002 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4003 return emulate_nm(ctxt);
4006 * Don't emulate a case that should never be hit, instead of working
4007 * around a lack of fxsave64/fxrstor64 on old compilers.
4009 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4010 return X86EMUL_UNHANDLEABLE;
4012 return X86EMUL_CONTINUE;
4016 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4017 * and restore MXCSR.
4019 static size_t __fxstate_size(int nregs)
4021 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4024 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4027 if (ctxt->mode == X86EMUL_MODE_PROT64)
4028 return __fxstate_size(16);
4030 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4031 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4035 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4038 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4039 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4041 * 3) 64-bit mode with REX.W prefix
4042 * - like (2), but XMM 8-15 are being saved and restored
4043 * 4) 64-bit mode without REX.W prefix
4044 * - like (3), but FIP and FDP are 64 bit
4046 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4047 * desired result. (4) is not emulated.
4049 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4050 * and FPU DS) should match.
4052 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4054 struct fxregs_state fx_state;
4057 rc = check_fxsr(ctxt);
4058 if (rc != X86EMUL_CONTINUE)
4063 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4067 if (rc != X86EMUL_CONTINUE)
4070 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4071 fxstate_size(ctxt));
4075 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4076 * in the host registers (via FXSAVE) instead, so they won't be modified.
4077 * (preemption has to stay disabled until FXRSTOR).
4079 * Use noinline to keep the stack for other functions called by callers small.
4081 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4082 const size_t used_size)
4084 struct fxregs_state fx_tmp;
4087 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4088 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4089 __fxstate_size(16) - used_size);
4094 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4096 struct fxregs_state fx_state;
4100 rc = check_fxsr(ctxt);
4101 if (rc != X86EMUL_CONTINUE)
4104 size = fxstate_size(ctxt);
4105 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4106 if (rc != X86EMUL_CONTINUE)
4111 if (size < __fxstate_size(16)) {
4112 rc = fxregs_fixup(&fx_state, size);
4113 if (rc != X86EMUL_CONTINUE)
4117 if (fx_state.mxcsr >> 16) {
4118 rc = emulate_gp(ctxt, 0);
4122 if (rc == X86EMUL_CONTINUE)
4123 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4131 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4135 eax = reg_read(ctxt, VCPU_REGS_RAX);
4136 edx = reg_read(ctxt, VCPU_REGS_RDX);
4137 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4139 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4140 return emulate_gp(ctxt, 0);
4142 return X86EMUL_CONTINUE;
4145 static bool valid_cr(int nr)
4157 static int check_cr_access(struct x86_emulate_ctxt *ctxt)
4159 if (!valid_cr(ctxt->modrm_reg))
4160 return emulate_ud(ctxt);
4162 return X86EMUL_CONTINUE;
4165 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4169 ctxt->ops->get_dr(ctxt, 7, &dr7);
4171 /* Check if DR7.Global_Enable is set */
4172 return dr7 & (1 << 13);
4175 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4177 int dr = ctxt->modrm_reg;
4181 return emulate_ud(ctxt);
4183 cr4 = ctxt->ops->get_cr(ctxt, 4);
4184 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4185 return emulate_ud(ctxt);
4187 if (check_dr7_gd(ctxt)) {
4190 ctxt->ops->get_dr(ctxt, 6, &dr6);
4191 dr6 &= ~DR_TRAP_BITS;
4192 dr6 |= DR6_BD | DR6_ACTIVE_LOW;
4193 ctxt->ops->set_dr(ctxt, 6, dr6);
4194 return emulate_db(ctxt);
4197 return X86EMUL_CONTINUE;
4200 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4202 u64 new_val = ctxt->src.val64;
4203 int dr = ctxt->modrm_reg;
4205 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4206 return emulate_gp(ctxt, 0);
4208 return check_dr_read(ctxt);
4211 static int check_svme(struct x86_emulate_ctxt *ctxt)
4215 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4217 if (!(efer & EFER_SVME))
4218 return emulate_ud(ctxt);
4220 return X86EMUL_CONTINUE;
4223 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4225 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4227 /* Valid physical address? */
4228 if (rax & 0xffff000000000000ULL)
4229 return emulate_gp(ctxt, 0);
4231 return check_svme(ctxt);
4234 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4236 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4238 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4239 return emulate_gp(ctxt, 0);
4241 return X86EMUL_CONTINUE;
4244 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4246 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4247 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4250 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4251 * in Ring3 when CR4.PCE=0.
4253 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4254 return X86EMUL_CONTINUE;
4257 * If CR4.PCE is set, the SDM requires CPL=0 or CR0.PE=0. The CR0.PE
4258 * check however is unnecessary because CPL is always 0 outside
4261 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4262 ctxt->ops->check_pmc(ctxt, rcx))
4263 return emulate_gp(ctxt, 0);
4265 return X86EMUL_CONTINUE;
4268 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4270 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4271 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4272 return emulate_gp(ctxt, 0);
4274 return X86EMUL_CONTINUE;
4277 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4279 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4280 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4281 return emulate_gp(ctxt, 0);
4283 return X86EMUL_CONTINUE;
4286 #define D(_y) { .flags = (_y) }
4287 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4288 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4289 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4290 #define N D(NotImpl)
4291 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4292 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4293 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4294 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4295 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4296 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4297 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4298 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4299 #define II(_f, _e, _i) \
4300 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4301 #define IIP(_f, _e, _i, _p) \
4302 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4303 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4304 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4306 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4307 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4308 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4309 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4310 #define I2bvIP(_f, _e, _i, _p) \
4311 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4313 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4314 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4315 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4317 static const struct opcode group7_rm0[] = {
4319 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4323 static const struct opcode group7_rm1[] = {
4324 DI(SrcNone | Priv, monitor),
4325 DI(SrcNone | Priv, mwait),
4329 static const struct opcode group7_rm2[] = {
4331 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4335 static const struct opcode group7_rm3[] = {
4336 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4337 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4338 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4339 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4340 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4341 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4342 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4343 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4346 static const struct opcode group7_rm7[] = {
4348 DIP(SrcNone, rdtscp, check_rdtsc),
4352 static const struct opcode group1[] = {
4354 F(Lock | PageTable, em_or),
4357 F(Lock | PageTable, em_and),
4363 static const struct opcode group1A[] = {
4364 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4367 static const struct opcode group2[] = {
4368 F(DstMem | ModRM, em_rol),
4369 F(DstMem | ModRM, em_ror),
4370 F(DstMem | ModRM, em_rcl),
4371 F(DstMem | ModRM, em_rcr),
4372 F(DstMem | ModRM, em_shl),
4373 F(DstMem | ModRM, em_shr),
4374 F(DstMem | ModRM, em_shl),
4375 F(DstMem | ModRM, em_sar),
4378 static const struct opcode group3[] = {
4379 F(DstMem | SrcImm | NoWrite, em_test),
4380 F(DstMem | SrcImm | NoWrite, em_test),
4381 F(DstMem | SrcNone | Lock, em_not),
4382 F(DstMem | SrcNone | Lock, em_neg),
4383 F(DstXacc | Src2Mem, em_mul_ex),
4384 F(DstXacc | Src2Mem, em_imul_ex),
4385 F(DstXacc | Src2Mem, em_div_ex),
4386 F(DstXacc | Src2Mem, em_idiv_ex),
4389 static const struct opcode group4[] = {
4390 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4391 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4395 static const struct opcode group5[] = {
4396 F(DstMem | SrcNone | Lock, em_inc),
4397 F(DstMem | SrcNone | Lock, em_dec),
4398 I(SrcMem | NearBranch | IsBranch, em_call_near_abs),
4399 I(SrcMemFAddr | ImplicitOps | IsBranch, em_call_far),
4400 I(SrcMem | NearBranch | IsBranch, em_jmp_abs),
4401 I(SrcMemFAddr | ImplicitOps | IsBranch, em_jmp_far),
4402 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4405 static const struct opcode group6[] = {
4406 II(Prot | DstMem, em_sldt, sldt),
4407 II(Prot | DstMem, em_str, str),
4408 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4409 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4413 static const struct group_dual group7 = { {
4414 II(Mov | DstMem, em_sgdt, sgdt),
4415 II(Mov | DstMem, em_sidt, sidt),
4416 II(SrcMem | Priv, em_lgdt, lgdt),
4417 II(SrcMem | Priv, em_lidt, lidt),
4418 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4419 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4420 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4426 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4427 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4431 static const struct opcode group8[] = {
4433 F(DstMem | SrcImmByte | NoWrite, em_bt),
4434 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4435 F(DstMem | SrcImmByte | Lock, em_btr),
4436 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4440 * The "memory" destination is actually always a register, since we come
4441 * from the register case of group9.
4443 static const struct gprefix pfx_0f_c7_7 = {
4444 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdpid),
4448 static const struct group_dual group9 = { {
4449 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4451 N, N, N, N, N, N, N,
4452 GP(0, &pfx_0f_c7_7),
4455 static const struct opcode group11[] = {
4456 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4460 static const struct gprefix pfx_0f_ae_7 = {
4461 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4464 static const struct group_dual group15 = { {
4465 I(ModRM | Aligned16, em_fxsave),
4466 I(ModRM | Aligned16, em_fxrstor),
4467 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4469 N, N, N, N, N, N, N, N,
4472 static const struct gprefix pfx_0f_6f_0f_7f = {
4473 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4476 static const struct instr_dual instr_dual_0f_2b = {
4480 static const struct gprefix pfx_0f_2b = {
4481 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4484 static const struct gprefix pfx_0f_10_0f_11 = {
4485 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4488 static const struct gprefix pfx_0f_28_0f_29 = {
4489 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4492 static const struct gprefix pfx_0f_e7 = {
4493 N, I(Sse, em_mov), N, N,
4496 static const struct escape escape_d9 = { {
4497 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4500 N, N, N, N, N, N, N, N,
4502 N, N, N, N, N, N, N, N,
4504 N, N, N, N, N, N, N, N,
4506 N, N, N, N, N, N, N, N,
4508 N, N, N, N, N, N, N, N,
4510 N, N, N, N, N, N, N, N,
4512 N, N, N, N, N, N, N, N,
4514 N, N, N, N, N, N, N, N,
4517 static const struct escape escape_db = { {
4518 N, N, N, N, N, N, N, N,
4521 N, N, N, N, N, N, N, N,
4523 N, N, N, N, N, N, N, N,
4525 N, N, N, N, N, N, N, N,
4527 N, N, N, N, N, N, N, N,
4529 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4531 N, N, N, N, N, N, N, N,
4533 N, N, N, N, N, N, N, N,
4535 N, N, N, N, N, N, N, N,
4538 static const struct escape escape_dd = { {
4539 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4542 N, N, N, N, N, N, N, N,
4544 N, N, N, N, N, N, N, N,
4546 N, N, N, N, N, N, N, N,
4548 N, N, N, N, N, N, N, N,
4550 N, N, N, N, N, N, N, N,
4552 N, N, N, N, N, N, N, N,
4554 N, N, N, N, N, N, N, N,
4556 N, N, N, N, N, N, N, N,
4559 static const struct instr_dual instr_dual_0f_c3 = {
4560 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4563 static const struct mode_dual mode_dual_63 = {
4564 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4567 static const struct instr_dual instr_dual_8d = {
4568 D(DstReg | SrcMem | ModRM | NoAccess), N
4571 static const struct opcode opcode_table[256] = {
4573 F6ALU(Lock, em_add),
4574 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4575 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4577 F6ALU(Lock | PageTable, em_or),
4578 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4581 F6ALU(Lock, em_adc),
4582 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4583 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4585 F6ALU(Lock, em_sbb),
4586 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4587 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4589 F6ALU(Lock | PageTable, em_and), N, N,
4591 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4593 F6ALU(Lock, em_xor), N, N,
4595 F6ALU(NoWrite, em_cmp), N, N,
4597 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4599 X8(I(SrcReg | Stack, em_push)),
4601 X8(I(DstReg | Stack, em_pop)),
4603 I(ImplicitOps | Stack | No64, em_pusha),
4604 I(ImplicitOps | Stack | No64, em_popa),
4605 N, MD(ModRM, &mode_dual_63),
4608 I(SrcImm | Mov | Stack, em_push),
4609 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4610 I(SrcImmByte | Mov | Stack, em_push),
4611 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4612 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4613 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4615 X16(D(SrcImmByte | NearBranch | IsBranch)),
4617 G(ByteOp | DstMem | SrcImm, group1),
4618 G(DstMem | SrcImm, group1),
4619 G(ByteOp | DstMem | SrcImm | No64, group1),
4620 G(DstMem | SrcImmByte, group1),
4621 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4622 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4624 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4625 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4626 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4627 ID(0, &instr_dual_8d),
4628 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4631 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4633 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4634 I(SrcImmFAddr | No64 | IsBranch, em_call_far), N,
4635 II(ImplicitOps | Stack, em_pushf, pushf),
4636 II(ImplicitOps | Stack, em_popf, popf),
4637 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4639 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4640 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4641 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4642 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4644 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4645 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4646 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4647 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4649 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4651 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4653 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4654 I(ImplicitOps | NearBranch | SrcImmU16 | IsBranch, em_ret_near_imm),
4655 I(ImplicitOps | NearBranch | IsBranch, em_ret),
4656 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4657 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4658 G(ByteOp, group11), G(0, group11),
4660 I(Stack | SrcImmU16 | Src2ImmByte | IsBranch, em_enter),
4661 I(Stack | IsBranch, em_leave),
4662 I(ImplicitOps | SrcImmU16 | IsBranch, em_ret_far_imm),
4663 I(ImplicitOps | IsBranch, em_ret_far),
4664 D(ImplicitOps | IsBranch), DI(SrcImmByte | IsBranch, intn),
4665 D(ImplicitOps | No64 | IsBranch),
4666 II(ImplicitOps | IsBranch, em_iret, iret),
4668 G(Src2One | ByteOp, group2), G(Src2One, group2),
4669 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4670 I(DstAcc | SrcImmUByte | No64, em_aam),
4671 I(DstAcc | SrcImmUByte | No64, em_aad),
4672 F(DstAcc | ByteOp | No64, em_salc),
4673 I(DstAcc | SrcXLat | ByteOp, em_mov),
4675 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4677 X3(I(SrcImmByte | NearBranch | IsBranch, em_loop)),
4678 I(SrcImmByte | NearBranch | IsBranch, em_jcxz),
4679 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4680 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4682 I(SrcImm | NearBranch | IsBranch, em_call),
4683 D(SrcImm | ImplicitOps | NearBranch | IsBranch),
4684 I(SrcImmFAddr | No64 | IsBranch, em_jmp_far),
4685 D(SrcImmByte | ImplicitOps | NearBranch | IsBranch),
4686 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4687 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4689 N, DI(ImplicitOps, icebp), N, N,
4690 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4691 G(ByteOp, group3), G(0, group3),
4693 D(ImplicitOps), D(ImplicitOps),
4694 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4695 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4698 static const struct opcode twobyte_table[256] = {
4700 G(0, group6), GD(0, &group7), N, N,
4701 N, I(ImplicitOps | EmulateOnUD | IsBranch, em_syscall),
4702 II(ImplicitOps | Priv, em_clts, clts), N,
4703 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4704 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4706 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4707 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4709 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4710 D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4711 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4712 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4713 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4714 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
4716 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access),
4717 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4718 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4720 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4723 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4724 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4725 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4728 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4729 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4730 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4731 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4732 I(ImplicitOps | EmulateOnUD | IsBranch, em_sysenter),
4733 I(ImplicitOps | Priv | EmulateOnUD | IsBranch, em_sysexit),
4735 N, N, N, N, N, N, N, N,
4737 X16(D(DstReg | SrcMem | ModRM)),
4739 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4744 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4749 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4751 X16(D(SrcImm | NearBranch | IsBranch)),
4753 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4755 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4756 II(ImplicitOps, em_cpuid, cpuid),
4757 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4758 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4759 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4761 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4762 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4763 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4764 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4765 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4766 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4768 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4769 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4770 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4771 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4772 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4773 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4777 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4778 I(DstReg | SrcMem | ModRM, em_bsf_c),
4779 I(DstReg | SrcMem | ModRM, em_bsr_c),
4780 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4782 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4783 N, ID(0, &instr_dual_0f_c3),
4784 N, N, N, GD(0, &group9),
4786 X8(I(DstReg, em_bswap)),
4788 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4790 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4791 N, N, N, N, N, N, N, N,
4793 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4796 static const struct instr_dual instr_dual_0f_38_f0 = {
4797 I(DstReg | SrcMem | Mov, em_movbe), N
4800 static const struct instr_dual instr_dual_0f_38_f1 = {
4801 I(DstMem | SrcReg | Mov, em_movbe), N
4804 static const struct gprefix three_byte_0f_38_f0 = {
4805 ID(0, &instr_dual_0f_38_f0), N, N, N
4808 static const struct gprefix three_byte_0f_38_f1 = {
4809 ID(0, &instr_dual_0f_38_f1), N, N, N
4813 * Insns below are selected by the prefix which indexed by the third opcode
4816 static const struct opcode opcode_map_0f_38[256] = {
4818 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4820 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4822 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4823 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4844 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4848 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4854 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4855 unsigned size, bool sign_extension)
4857 int rc = X86EMUL_CONTINUE;
4861 op->addr.mem.ea = ctxt->_eip;
4862 /* NB. Immediates are sign-extended as necessary. */
4863 switch (op->bytes) {
4865 op->val = insn_fetch(s8, ctxt);
4868 op->val = insn_fetch(s16, ctxt);
4871 op->val = insn_fetch(s32, ctxt);
4874 op->val = insn_fetch(s64, ctxt);
4877 if (!sign_extension) {
4878 switch (op->bytes) {
4886 op->val &= 0xffffffff;
4894 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4897 int rc = X86EMUL_CONTINUE;
4901 decode_register_operand(ctxt, op);
4904 rc = decode_imm(ctxt, op, 1, false);
4907 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4911 if (ctxt->d & BitOp)
4912 fetch_bit_operand(ctxt);
4913 op->orig_val = op->val;
4916 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4920 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4921 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4922 fetch_register_operand(op);
4923 op->orig_val = op->val;
4927 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4928 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4929 fetch_register_operand(op);
4930 op->orig_val = op->val;
4933 if (ctxt->d & ByteOp) {
4938 op->bytes = ctxt->op_bytes;
4939 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4940 fetch_register_operand(op);
4941 op->orig_val = op->val;
4945 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4947 register_address(ctxt, VCPU_REGS_RDI);
4948 op->addr.mem.seg = VCPU_SREG_ES;
4955 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4956 fetch_register_operand(op);
4961 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4964 rc = decode_imm(ctxt, op, 1, true);
4972 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4975 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4978 ctxt->memop.bytes = 1;
4979 if (ctxt->memop.type == OP_REG) {
4980 ctxt->memop.addr.reg = decode_register(ctxt,
4981 ctxt->modrm_rm, true);
4982 fetch_register_operand(&ctxt->memop);
4986 ctxt->memop.bytes = 2;
4989 ctxt->memop.bytes = 4;
4992 rc = decode_imm(ctxt, op, 2, false);
4995 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4999 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5001 register_address(ctxt, VCPU_REGS_RSI);
5002 op->addr.mem.seg = ctxt->seg_override;
5008 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5011 reg_read(ctxt, VCPU_REGS_RBX) +
5012 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5013 op->addr.mem.seg = ctxt->seg_override;
5018 op->addr.mem.ea = ctxt->_eip;
5019 op->bytes = ctxt->op_bytes + 2;
5020 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5023 ctxt->memop.bytes = ctxt->op_bytes + 2;
5027 op->val = VCPU_SREG_ES;
5031 op->val = VCPU_SREG_CS;
5035 op->val = VCPU_SREG_SS;
5039 op->val = VCPU_SREG_DS;
5043 op->val = VCPU_SREG_FS;
5047 op->val = VCPU_SREG_GS;
5050 /* Special instructions do their own operand decoding. */
5052 op->type = OP_NONE; /* Disable writeback. */
5060 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type)
5062 int rc = X86EMUL_CONTINUE;
5063 int mode = ctxt->mode;
5064 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5065 bool op_prefix = false;
5066 bool has_seg_override = false;
5067 struct opcode opcode;
5069 struct desc_struct desc;
5071 ctxt->memop.type = OP_NONE;
5072 ctxt->memopp = NULL;
5073 ctxt->_eip = ctxt->eip;
5074 ctxt->fetch.ptr = ctxt->fetch.data;
5075 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5076 ctxt->opcode_len = 1;
5077 ctxt->intercept = x86_intercept_none;
5079 memcpy(ctxt->fetch.data, insn, insn_len);
5081 rc = __do_insn_fetch_bytes(ctxt, 1);
5082 if (rc != X86EMUL_CONTINUE)
5087 case X86EMUL_MODE_REAL:
5088 case X86EMUL_MODE_VM86:
5089 def_op_bytes = def_ad_bytes = 2;
5090 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5092 def_op_bytes = def_ad_bytes = 4;
5094 case X86EMUL_MODE_PROT16:
5095 def_op_bytes = def_ad_bytes = 2;
5097 case X86EMUL_MODE_PROT32:
5098 def_op_bytes = def_ad_bytes = 4;
5100 #ifdef CONFIG_X86_64
5101 case X86EMUL_MODE_PROT64:
5107 return EMULATION_FAILED;
5110 ctxt->op_bytes = def_op_bytes;
5111 ctxt->ad_bytes = def_ad_bytes;
5113 /* Legacy prefixes. */
5115 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5116 case 0x66: /* operand-size override */
5118 /* switch between 2/4 bytes */
5119 ctxt->op_bytes = def_op_bytes ^ 6;
5121 case 0x67: /* address-size override */
5122 if (mode == X86EMUL_MODE_PROT64)
5123 /* switch between 4/8 bytes */
5124 ctxt->ad_bytes = def_ad_bytes ^ 12;
5126 /* switch between 2/4 bytes */
5127 ctxt->ad_bytes = def_ad_bytes ^ 6;
5129 case 0x26: /* ES override */
5130 has_seg_override = true;
5131 ctxt->seg_override = VCPU_SREG_ES;
5133 case 0x2e: /* CS override */
5134 has_seg_override = true;
5135 ctxt->seg_override = VCPU_SREG_CS;
5137 case 0x36: /* SS override */
5138 has_seg_override = true;
5139 ctxt->seg_override = VCPU_SREG_SS;
5141 case 0x3e: /* DS override */
5142 has_seg_override = true;
5143 ctxt->seg_override = VCPU_SREG_DS;
5145 case 0x64: /* FS override */
5146 has_seg_override = true;
5147 ctxt->seg_override = VCPU_SREG_FS;
5149 case 0x65: /* GS override */
5150 has_seg_override = true;
5151 ctxt->seg_override = VCPU_SREG_GS;
5153 case 0x40 ... 0x4f: /* REX */
5154 if (mode != X86EMUL_MODE_PROT64)
5156 ctxt->rex_prefix = ctxt->b;
5158 case 0xf0: /* LOCK */
5159 ctxt->lock_prefix = 1;
5161 case 0xf2: /* REPNE/REPNZ */
5162 case 0xf3: /* REP/REPE/REPZ */
5163 ctxt->rep_prefix = ctxt->b;
5169 /* Any legacy prefix after a REX prefix nullifies its effect. */
5171 ctxt->rex_prefix = 0;
5177 if (ctxt->rex_prefix & 8)
5178 ctxt->op_bytes = 8; /* REX.W */
5180 /* Opcode byte(s). */
5181 opcode = opcode_table[ctxt->b];
5182 /* Two-byte opcode? */
5183 if (ctxt->b == 0x0f) {
5184 ctxt->opcode_len = 2;
5185 ctxt->b = insn_fetch(u8, ctxt);
5186 opcode = twobyte_table[ctxt->b];
5188 /* 0F_38 opcode map */
5189 if (ctxt->b == 0x38) {
5190 ctxt->opcode_len = 3;
5191 ctxt->b = insn_fetch(u8, ctxt);
5192 opcode = opcode_map_0f_38[ctxt->b];
5195 ctxt->d = opcode.flags;
5197 if (ctxt->d & ModRM)
5198 ctxt->modrm = insn_fetch(u8, ctxt);
5200 /* vex-prefix instructions are not implemented */
5201 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5202 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5206 while (ctxt->d & GroupMask) {
5207 switch (ctxt->d & GroupMask) {
5209 goffset = (ctxt->modrm >> 3) & 7;
5210 opcode = opcode.u.group[goffset];
5213 goffset = (ctxt->modrm >> 3) & 7;
5214 if ((ctxt->modrm >> 6) == 3)
5215 opcode = opcode.u.gdual->mod3[goffset];
5217 opcode = opcode.u.gdual->mod012[goffset];
5220 goffset = ctxt->modrm & 7;
5221 opcode = opcode.u.group[goffset];
5224 if (ctxt->rep_prefix && op_prefix)
5225 return EMULATION_FAILED;
5226 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5227 switch (simd_prefix) {
5228 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5229 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5230 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5231 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5235 if (ctxt->modrm > 0xbf) {
5236 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5237 u32 index = array_index_nospec(
5238 ctxt->modrm - 0xc0, size);
5240 opcode = opcode.u.esc->high[index];
5242 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5246 if ((ctxt->modrm >> 6) == 3)
5247 opcode = opcode.u.idual->mod3;
5249 opcode = opcode.u.idual->mod012;
5252 if (ctxt->mode == X86EMUL_MODE_PROT64)
5253 opcode = opcode.u.mdual->mode64;
5255 opcode = opcode.u.mdual->mode32;
5258 return EMULATION_FAILED;
5261 ctxt->d &= ~(u64)GroupMask;
5262 ctxt->d |= opcode.flags;
5265 ctxt->is_branch = opcode.flags & IsBranch;
5269 return EMULATION_FAILED;
5271 ctxt->execute = opcode.u.execute;
5273 if (unlikely(emulation_type & EMULTYPE_TRAP_UD) &&
5274 likely(!(ctxt->d & EmulateOnUD)))
5275 return EMULATION_FAILED;
5277 if (unlikely(ctxt->d &
5278 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5281 * These are copied unconditionally here, and checked unconditionally
5282 * in x86_emulate_insn.
5284 ctxt->check_perm = opcode.check_perm;
5285 ctxt->intercept = opcode.intercept;
5287 if (ctxt->d & NotImpl)
5288 return EMULATION_FAILED;
5290 if (mode == X86EMUL_MODE_PROT64) {
5291 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5293 else if (ctxt->d & NearBranch)
5297 if (ctxt->d & Op3264) {
5298 if (mode == X86EMUL_MODE_PROT64)
5304 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5308 ctxt->op_bytes = 16;
5309 else if (ctxt->d & Mmx)
5313 /* ModRM and SIB bytes. */
5314 if (ctxt->d & ModRM) {
5315 rc = decode_modrm(ctxt, &ctxt->memop);
5316 if (!has_seg_override) {
5317 has_seg_override = true;
5318 ctxt->seg_override = ctxt->modrm_seg;
5320 } else if (ctxt->d & MemAbs)
5321 rc = decode_abs(ctxt, &ctxt->memop);
5322 if (rc != X86EMUL_CONTINUE)
5325 if (!has_seg_override)
5326 ctxt->seg_override = VCPU_SREG_DS;
5328 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5331 * Decode and fetch the source operand: register, memory
5334 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5335 if (rc != X86EMUL_CONTINUE)
5339 * Decode and fetch the second source operand: register, memory
5342 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5343 if (rc != X86EMUL_CONTINUE)
5346 /* Decode and fetch the destination operand: register or memory. */
5347 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5349 if (ctxt->rip_relative && likely(ctxt->memopp))
5350 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5351 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5354 if (rc == X86EMUL_PROPAGATE_FAULT)
5355 ctxt->have_exception = true;
5356 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5359 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5361 return ctxt->d & PageTable;
5364 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5366 /* The second termination condition only applies for REPE
5367 * and REPNE. Test if the repeat string operation prefix is
5368 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5369 * corresponding termination condition according to:
5370 * - if REPE/REPZ and ZF = 0 then done
5371 * - if REPNE/REPNZ and ZF = 1 then done
5373 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5374 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5375 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5376 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5377 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5378 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5384 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5389 rc = asm_safe("fwait");
5392 if (unlikely(rc != X86EMUL_CONTINUE))
5393 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5395 return X86EMUL_CONTINUE;
5398 static void fetch_possible_mmx_operand(struct operand *op)
5400 if (op->type == OP_MM)
5401 kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5404 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5406 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5408 if (!(ctxt->d & ByteOp))
5409 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5411 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5412 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5413 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5414 : "c"(ctxt->src2.val));
5416 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5417 if (!fop) /* exception is returned in fop variable */
5418 return emulate_de(ctxt);
5419 return X86EMUL_CONTINUE;
5422 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5424 /* Clear fields that are set conditionally but read without a guard. */
5425 ctxt->rip_relative = false;
5426 ctxt->rex_prefix = 0;
5427 ctxt->lock_prefix = 0;
5428 ctxt->rep_prefix = 0;
5429 ctxt->regs_valid = 0;
5430 ctxt->regs_dirty = 0;
5432 ctxt->io_read.pos = 0;
5433 ctxt->io_read.end = 0;
5434 ctxt->mem_read.end = 0;
5437 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5439 const struct x86_emulate_ops *ops = ctxt->ops;
5440 int rc = X86EMUL_CONTINUE;
5441 int saved_dst_type = ctxt->dst.type;
5442 unsigned emul_flags;
5444 ctxt->mem_read.pos = 0;
5446 /* LOCK prefix is allowed only with some instructions */
5447 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5448 rc = emulate_ud(ctxt);
5452 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5453 rc = emulate_ud(ctxt);
5457 emul_flags = ctxt->ops->get_hflags(ctxt);
5458 if (unlikely(ctxt->d &
5459 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5460 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5461 (ctxt->d & Undefined)) {
5462 rc = emulate_ud(ctxt);
5466 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5467 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5468 rc = emulate_ud(ctxt);
5472 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5473 rc = emulate_nm(ctxt);
5477 if (ctxt->d & Mmx) {
5478 rc = flush_pending_x87_faults(ctxt);
5479 if (rc != X86EMUL_CONTINUE)
5482 * Now that we know the fpu is exception safe, we can fetch
5485 fetch_possible_mmx_operand(&ctxt->src);
5486 fetch_possible_mmx_operand(&ctxt->src2);
5487 if (!(ctxt->d & Mov))
5488 fetch_possible_mmx_operand(&ctxt->dst);
5491 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5492 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5493 X86_ICPT_PRE_EXCEPT);
5494 if (rc != X86EMUL_CONTINUE)
5498 /* Instruction can only be executed in protected mode */
5499 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5500 rc = emulate_ud(ctxt);
5504 /* Privileged instruction can be executed only in CPL=0 */
5505 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5506 if (ctxt->d & PrivUD)
5507 rc = emulate_ud(ctxt);
5509 rc = emulate_gp(ctxt, 0);
5513 /* Do instruction specific permission checks */
5514 if (ctxt->d & CheckPerm) {
5515 rc = ctxt->check_perm(ctxt);
5516 if (rc != X86EMUL_CONTINUE)
5520 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5521 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5522 X86_ICPT_POST_EXCEPT);
5523 if (rc != X86EMUL_CONTINUE)
5527 if (ctxt->rep_prefix && (ctxt->d & String)) {
5528 /* All REP prefixes have the same first termination condition */
5529 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5530 string_registers_quirk(ctxt);
5531 ctxt->eip = ctxt->_eip;
5532 ctxt->eflags &= ~X86_EFLAGS_RF;
5538 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5539 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5540 ctxt->src.valptr, ctxt->src.bytes);
5541 if (rc != X86EMUL_CONTINUE)
5543 ctxt->src.orig_val64 = ctxt->src.val64;
5546 if (ctxt->src2.type == OP_MEM) {
5547 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5548 &ctxt->src2.val, ctxt->src2.bytes);
5549 if (rc != X86EMUL_CONTINUE)
5553 if ((ctxt->d & DstMask) == ImplicitOps)
5557 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5558 /* optimisation - avoid slow emulated read if Mov */
5559 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5560 &ctxt->dst.val, ctxt->dst.bytes);
5561 if (rc != X86EMUL_CONTINUE) {
5562 if (!(ctxt->d & NoWrite) &&
5563 rc == X86EMUL_PROPAGATE_FAULT &&
5564 ctxt->exception.vector == PF_VECTOR)
5565 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5569 /* Copy full 64-bit value for CMPXCHG8B. */
5570 ctxt->dst.orig_val64 = ctxt->dst.val64;
5574 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5575 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5576 X86_ICPT_POST_MEMACCESS);
5577 if (rc != X86EMUL_CONTINUE)
5581 if (ctxt->rep_prefix && (ctxt->d & String))
5582 ctxt->eflags |= X86_EFLAGS_RF;
5584 ctxt->eflags &= ~X86_EFLAGS_RF;
5586 if (ctxt->execute) {
5587 if (ctxt->d & Fastop)
5588 rc = fastop(ctxt, ctxt->fop);
5590 rc = ctxt->execute(ctxt);
5591 if (rc != X86EMUL_CONTINUE)
5596 if (ctxt->opcode_len == 2)
5598 else if (ctxt->opcode_len == 3)
5599 goto threebyte_insn;
5602 case 0x70 ... 0x7f: /* jcc (short) */
5603 if (test_cc(ctxt->b, ctxt->eflags))
5604 rc = jmp_rel(ctxt, ctxt->src.val);
5606 case 0x8d: /* lea r16/r32, m */
5607 ctxt->dst.val = ctxt->src.addr.mem.ea;
5609 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5610 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5611 ctxt->dst.type = OP_NONE;
5615 case 0x98: /* cbw/cwde/cdqe */
5616 switch (ctxt->op_bytes) {
5617 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5618 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5619 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5622 case 0xcc: /* int3 */
5623 rc = emulate_int(ctxt, 3);
5625 case 0xcd: /* int n */
5626 rc = emulate_int(ctxt, ctxt->src.val);
5628 case 0xce: /* into */
5629 if (ctxt->eflags & X86_EFLAGS_OF)
5630 rc = emulate_int(ctxt, 4);
5632 case 0xe9: /* jmp rel */
5633 case 0xeb: /* jmp rel short */
5634 rc = jmp_rel(ctxt, ctxt->src.val);
5635 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5637 case 0xf4: /* hlt */
5638 ctxt->ops->halt(ctxt);
5640 case 0xf5: /* cmc */
5641 /* complement carry flag from eflags reg */
5642 ctxt->eflags ^= X86_EFLAGS_CF;
5644 case 0xf8: /* clc */
5645 ctxt->eflags &= ~X86_EFLAGS_CF;
5647 case 0xf9: /* stc */
5648 ctxt->eflags |= X86_EFLAGS_CF;
5650 case 0xfc: /* cld */
5651 ctxt->eflags &= ~X86_EFLAGS_DF;
5653 case 0xfd: /* std */
5654 ctxt->eflags |= X86_EFLAGS_DF;
5657 goto cannot_emulate;
5660 if (rc != X86EMUL_CONTINUE)
5664 if (ctxt->d & SrcWrite) {
5665 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5666 rc = writeback(ctxt, &ctxt->src);
5667 if (rc != X86EMUL_CONTINUE)
5670 if (!(ctxt->d & NoWrite)) {
5671 rc = writeback(ctxt, &ctxt->dst);
5672 if (rc != X86EMUL_CONTINUE)
5677 * restore dst type in case the decoding will be reused
5678 * (happens for string instruction )
5680 ctxt->dst.type = saved_dst_type;
5682 if ((ctxt->d & SrcMask) == SrcSI)
5683 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5685 if ((ctxt->d & DstMask) == DstDI)
5686 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5688 if (ctxt->rep_prefix && (ctxt->d & String)) {
5690 struct read_cache *r = &ctxt->io_read;
5691 if ((ctxt->d & SrcMask) == SrcSI)
5692 count = ctxt->src.count;
5694 count = ctxt->dst.count;
5695 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5697 if (!string_insn_completed(ctxt)) {
5699 * Re-enter guest when pio read ahead buffer is empty
5700 * or, if it is not used, after each 1024 iteration.
5702 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5703 (r->end == 0 || r->end != r->pos)) {
5705 * Reset read cache. Usually happens before
5706 * decode, but since instruction is restarted
5707 * we have to do it here.
5709 ctxt->mem_read.end = 0;
5710 writeback_registers(ctxt);
5711 return EMULATION_RESTART;
5713 goto done; /* skip rip writeback */
5715 ctxt->eflags &= ~X86_EFLAGS_RF;
5718 ctxt->eip = ctxt->_eip;
5719 if (ctxt->mode != X86EMUL_MODE_PROT64)
5720 ctxt->eip = (u32)ctxt->_eip;
5723 if (rc == X86EMUL_PROPAGATE_FAULT) {
5724 if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt))
5725 return EMULATION_FAILED;
5726 ctxt->have_exception = true;
5728 if (rc == X86EMUL_INTERCEPTED)
5729 return EMULATION_INTERCEPTED;
5731 if (rc == X86EMUL_CONTINUE)
5732 writeback_registers(ctxt);
5734 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5738 case 0x09: /* wbinvd */
5739 (ctxt->ops->wbinvd)(ctxt);
5741 case 0x08: /* invd */
5742 case 0x0d: /* GrpP (prefetch) */
5743 case 0x18: /* Grp16 (prefetch/nop) */
5744 case 0x1f: /* nop */
5746 case 0x20: /* mov cr, reg */
5747 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5749 case 0x21: /* mov from dr to reg */
5750 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5752 case 0x40 ... 0x4f: /* cmov */
5753 if (test_cc(ctxt->b, ctxt->eflags))
5754 ctxt->dst.val = ctxt->src.val;
5755 else if (ctxt->op_bytes != 4)
5756 ctxt->dst.type = OP_NONE; /* no writeback */
5758 case 0x80 ... 0x8f: /* jnz rel, etc*/
5759 if (test_cc(ctxt->b, ctxt->eflags))
5760 rc = jmp_rel(ctxt, ctxt->src.val);
5762 case 0x90 ... 0x9f: /* setcc r/m8 */
5763 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5765 case 0xb6 ... 0xb7: /* movzx */
5766 ctxt->dst.bytes = ctxt->op_bytes;
5767 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5768 : (u16) ctxt->src.val;
5770 case 0xbe ... 0xbf: /* movsx */
5771 ctxt->dst.bytes = ctxt->op_bytes;
5772 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5773 (s16) ctxt->src.val;
5776 goto cannot_emulate;
5781 if (rc != X86EMUL_CONTINUE)
5787 return EMULATION_FAILED;
5790 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5792 invalidate_registers(ctxt);
5795 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5797 writeback_registers(ctxt);
5800 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5802 if (ctxt->rep_prefix && (ctxt->d & String))
5805 if (ctxt->d & TwoMemOp)