1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
165 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
167 #define X2(x...) x, x
168 #define X3(x...) X2(x), x
169 #define X4(x...) X2(x), X2(x)
170 #define X5(x...) X4(x), x
171 #define X6(x...) X4(x), X2(x)
172 #define X7(x...) X4(x), X3(x)
173 #define X8(x...) X4(x), X4(x)
174 #define X16(x...) X8(x), X8(x)
176 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
177 #define FASTOP_SIZE 8
180 * fastop functions have a special calling convention:
185 * flags: rflags (in/out)
186 * ex: rsi (in:fastop pointer, out:zero if exception)
188 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
189 * different operand sizes can be reached by calculation, rather than a jump
190 * table (which would be bigger than the code).
192 * fastop functions are declared as taking a never-defined fastop parameter,
193 * so they can't be called from C directly.
202 int (*execute)(struct x86_emulate_ctxt *ctxt);
203 const struct opcode *group;
204 const struct group_dual *gdual;
205 const struct gprefix *gprefix;
206 const struct escape *esc;
207 void (*fastop)(struct fastop *fake);
209 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
213 struct opcode mod012[8];
214 struct opcode mod3[8];
218 struct opcode pfx_no;
219 struct opcode pfx_66;
220 struct opcode pfx_f2;
221 struct opcode pfx_f3;
226 struct opcode high[64];
229 /* EFLAGS bit definitions. */
230 #define EFLG_ID (1<<21)
231 #define EFLG_VIP (1<<20)
232 #define EFLG_VIF (1<<19)
233 #define EFLG_AC (1<<18)
234 #define EFLG_VM (1<<17)
235 #define EFLG_RF (1<<16)
236 #define EFLG_IOPL (3<<12)
237 #define EFLG_NT (1<<14)
238 #define EFLG_OF (1<<11)
239 #define EFLG_DF (1<<10)
240 #define EFLG_IF (1<<9)
241 #define EFLG_TF (1<<8)
242 #define EFLG_SF (1<<7)
243 #define EFLG_ZF (1<<6)
244 #define EFLG_AF (1<<4)
245 #define EFLG_PF (1<<2)
246 #define EFLG_CF (1<<0)
248 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
249 #define EFLG_RESERVED_ONE_MASK 2
251 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 if (!(ctxt->regs_valid & (1 << nr))) {
254 ctxt->regs_valid |= 1 << nr;
255 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
257 return ctxt->_regs[nr];
260 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
262 ctxt->regs_valid |= 1 << nr;
263 ctxt->regs_dirty |= 1 << nr;
264 return &ctxt->_regs[nr];
267 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
270 return reg_write(ctxt, nr);
273 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
277 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
278 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
281 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
283 ctxt->regs_dirty = 0;
284 ctxt->regs_valid = 0;
288 * These EFLAGS bits are restored from saved value during emulation, and
289 * any changes are written back to the saved value after emulation.
291 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
301 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
302 #define FOP_RET "ret \n\t"
304 #define FOP_START(op) \
305 extern void em_##op(struct fastop *fake); \
306 asm(".pushsection .text, \"ax\" \n\t" \
307 ".global em_" #op " \n\t" \
314 #define FOPNOP() FOP_ALIGN FOP_RET
316 #define FOP1E(op, dst) \
317 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
319 #define FOP1EEX(op, dst) \
320 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
322 #define FASTOP1(op) \
327 ON64(FOP1E(op##q, rax)) \
330 /* 1-operand, using src2 (for MUL/DIV r/m) */
331 #define FASTOP1SRC2(op, name) \
336 ON64(FOP1E(op, rcx)) \
339 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
340 #define FASTOP1SRC2EX(op, name) \
345 ON64(FOP1EEX(op, rcx)) \
348 #define FOP2E(op, dst, src) \
349 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
351 #define FASTOP2(op) \
353 FOP2E(op##b, al, dl) \
354 FOP2E(op##w, ax, dx) \
355 FOP2E(op##l, eax, edx) \
356 ON64(FOP2E(op##q, rax, rdx)) \
359 /* 2 operand, word only */
360 #define FASTOP2W(op) \
363 FOP2E(op##w, ax, dx) \
364 FOP2E(op##l, eax, edx) \
365 ON64(FOP2E(op##q, rax, rdx)) \
368 /* 2 operand, src is CL */
369 #define FASTOP2CL(op) \
371 FOP2E(op##b, al, cl) \
372 FOP2E(op##w, ax, cl) \
373 FOP2E(op##l, eax, cl) \
374 ON64(FOP2E(op##q, rax, cl)) \
377 #define FOP3E(op, dst, src, src2) \
378 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
380 /* 3-operand, word-only, src2=cl */
381 #define FASTOP3WCL(op) \
384 FOP3E(op##w, ax, dx, cl) \
385 FOP3E(op##l, eax, edx, cl) \
386 ON64(FOP3E(op##q, rax, rdx, cl)) \
389 /* Special case for SETcc - 1 instruction per cc */
390 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
392 asm(".global kvm_fastop_exception \n"
393 "kvm_fastop_exception: xor %esi, %esi; ret");
414 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
417 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
418 enum x86_intercept intercept,
419 enum x86_intercept_stage stage)
421 struct x86_instruction_info info = {
422 .intercept = intercept,
423 .rep_prefix = ctxt->rep_prefix,
424 .modrm_mod = ctxt->modrm_mod,
425 .modrm_reg = ctxt->modrm_reg,
426 .modrm_rm = ctxt->modrm_rm,
427 .src_val = ctxt->src.val64,
428 .src_bytes = ctxt->src.bytes,
429 .dst_bytes = ctxt->dst.bytes,
430 .ad_bytes = ctxt->ad_bytes,
431 .next_rip = ctxt->eip,
434 return ctxt->ops->intercept(ctxt, &info, stage);
437 static void assign_masked(ulong *dest, ulong src, ulong mask)
439 *dest = (*dest & ~mask) | (src & mask);
442 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
444 return (1UL << (ctxt->ad_bytes << 3)) - 1;
447 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
450 struct desc_struct ss;
452 if (ctxt->mode == X86EMUL_MODE_PROT64)
454 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
455 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
458 static int stack_size(struct x86_emulate_ctxt *ctxt)
460 return (__fls(stack_mask(ctxt)) + 1) >> 3;
463 /* Access/update address held in a register, based on addressing mode. */
464 static inline unsigned long
465 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
467 if (ctxt->ad_bytes == sizeof(unsigned long))
470 return reg & ad_mask(ctxt);
473 static inline unsigned long
474 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
476 return address_mask(ctxt, reg);
479 static void masked_increment(ulong *reg, ulong mask, int inc)
481 assign_masked(reg, *reg + inc, mask);
485 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
489 if (ctxt->ad_bytes == sizeof(unsigned long))
492 mask = ad_mask(ctxt);
493 masked_increment(reg, mask, inc);
496 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
498 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
501 static u32 desc_limit_scaled(struct desc_struct *desc)
503 u32 limit = get_desc_limit(desc);
505 return desc->g ? (limit << 12) | 0xfff : limit;
508 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
510 ctxt->has_seg_override = true;
511 ctxt->seg_override = seg;
514 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
516 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
519 return ctxt->ops->get_cached_segment_base(ctxt, seg);
522 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
524 if (!ctxt->has_seg_override)
527 return ctxt->seg_override;
530 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
531 u32 error, bool valid)
533 ctxt->exception.vector = vec;
534 ctxt->exception.error_code = error;
535 ctxt->exception.error_code_valid = valid;
536 return X86EMUL_PROPAGATE_FAULT;
539 static int emulate_db(struct x86_emulate_ctxt *ctxt)
541 return emulate_exception(ctxt, DB_VECTOR, 0, false);
544 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
546 return emulate_exception(ctxt, GP_VECTOR, err, true);
549 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
551 return emulate_exception(ctxt, SS_VECTOR, err, true);
554 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
556 return emulate_exception(ctxt, UD_VECTOR, 0, false);
559 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
561 return emulate_exception(ctxt, TS_VECTOR, err, true);
564 static int emulate_de(struct x86_emulate_ctxt *ctxt)
566 return emulate_exception(ctxt, DE_VECTOR, 0, false);
569 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
571 return emulate_exception(ctxt, NM_VECTOR, 0, false);
574 static inline void assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
576 switch (ctxt->op_bytes) {
578 ctxt->_eip = (u16)dst;
581 ctxt->_eip = (u32)dst;
587 WARN(1, "unsupported eip assignment size\n");
591 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
593 assign_eip_near(ctxt, ctxt->_eip + rel);
596 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
599 struct desc_struct desc;
601 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
605 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
610 struct desc_struct desc;
612 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
613 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
617 * x86 defines three classes of vector instructions: explicitly
618 * aligned, explicitly unaligned, and the rest, which change behaviour
619 * depending on whether they're AVX encoded or not.
621 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
622 * subject to the same check.
624 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
626 if (likely(size < 16))
629 if (ctxt->d & Aligned)
631 else if (ctxt->d & Unaligned)
633 else if (ctxt->d & Avx)
639 static int __linearize(struct x86_emulate_ctxt *ctxt,
640 struct segmented_address addr,
641 unsigned size, bool write, bool fetch,
644 struct desc_struct desc;
651 la = seg_base(ctxt, addr.seg) + addr.ea;
652 switch (ctxt->mode) {
653 case X86EMUL_MODE_PROT64:
654 if (((signed long)la << 16) >> 16 != la)
655 return emulate_gp(ctxt, 0);
658 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
662 /* code segment in protected mode or read-only data segment */
663 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
664 || !(desc.type & 2)) && write)
666 /* unreadable code segment */
667 if (!fetch && (desc.type & 8) && !(desc.type & 2))
669 lim = desc_limit_scaled(&desc);
670 if ((desc.type & 8) || !(desc.type & 4)) {
671 /* expand-up segment */
672 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
675 /* expand-down segment */
676 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
678 lim = desc.d ? 0xffffffff : 0xffff;
679 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
682 cpl = ctxt->ops->cpl(ctxt);
683 if (!(desc.type & 8)) {
687 } else if ((desc.type & 8) && !(desc.type & 4)) {
688 /* nonconforming code segment */
691 } else if ((desc.type & 8) && (desc.type & 4)) {
692 /* conforming code segment */
698 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
700 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
701 return emulate_gp(ctxt, 0);
703 return X86EMUL_CONTINUE;
705 if (addr.seg == VCPU_SREG_SS)
706 return emulate_ss(ctxt, sel);
708 return emulate_gp(ctxt, sel);
711 static int linearize(struct x86_emulate_ctxt *ctxt,
712 struct segmented_address addr,
713 unsigned size, bool write,
716 return __linearize(ctxt, addr, size, write, false, linear);
720 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
721 struct segmented_address addr,
728 rc = linearize(ctxt, addr, size, false, &linear);
729 if (rc != X86EMUL_CONTINUE)
731 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
735 * Fetch the next byte of the instruction being emulated which is pointed to
736 * by ctxt->_eip, then increment ctxt->_eip.
738 * Also prefetch the remaining bytes of the instruction without crossing page
739 * boundary if they are not in fetch_cache yet.
741 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
743 struct fetch_cache *fc = &ctxt->fetch;
747 if (ctxt->_eip == fc->end) {
748 unsigned long linear;
749 struct segmented_address addr = { .seg = VCPU_SREG_CS,
751 cur_size = fc->end - fc->start;
752 size = min(15UL - cur_size,
753 PAGE_SIZE - offset_in_page(ctxt->_eip));
754 rc = __linearize(ctxt, addr, size, false, true, &linear);
755 if (unlikely(rc != X86EMUL_CONTINUE))
757 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
758 size, &ctxt->exception);
759 if (unlikely(rc != X86EMUL_CONTINUE))
763 *dest = fc->data[ctxt->_eip - fc->start];
765 return X86EMUL_CONTINUE;
768 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
769 void *dest, unsigned size)
773 /* x86 instructions are limited to 15 bytes. */
774 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
775 return X86EMUL_UNHANDLEABLE;
777 rc = do_insn_fetch_byte(ctxt, dest++);
778 if (rc != X86EMUL_CONTINUE)
781 return X86EMUL_CONTINUE;
784 /* Fetch next part of the instruction being emulated. */
785 #define insn_fetch(_type, _ctxt) \
786 ({ unsigned long _x; \
787 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
788 if (rc != X86EMUL_CONTINUE) \
793 #define insn_fetch_arr(_arr, _size, _ctxt) \
794 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
795 if (rc != X86EMUL_CONTINUE) \
800 * Given the 'reg' portion of a ModRM byte, and a register block, return a
801 * pointer into the block that addresses the relevant register.
802 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
804 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
808 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
810 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
811 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
813 p = reg_rmw(ctxt, modrm_reg);
817 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
818 struct segmented_address addr,
819 u16 *size, unsigned long *address, int op_bytes)
826 rc = segmented_read_std(ctxt, addr, size, 2);
827 if (rc != X86EMUL_CONTINUE)
830 rc = segmented_read_std(ctxt, addr, address, op_bytes);
844 FASTOP1SRC2(mul, mul_ex);
845 FASTOP1SRC2(imul, imul_ex);
846 FASTOP1SRC2EX(div, div_ex);
847 FASTOP1SRC2EX(idiv, idiv_ex);
876 static u8 test_cc(unsigned int condition, unsigned long flags)
879 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
881 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
882 asm("push %[flags]; popf; call *%[fastop]"
883 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
887 static void fetch_register_operand(struct operand *op)
891 op->val = *(u8 *)op->addr.reg;
894 op->val = *(u16 *)op->addr.reg;
897 op->val = *(u32 *)op->addr.reg;
900 op->val = *(u64 *)op->addr.reg;
905 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
907 ctxt->ops->get_fpu(ctxt);
909 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
910 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
911 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
912 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
913 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
914 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
915 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
916 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
918 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
919 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
920 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
921 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
922 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
923 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
924 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
925 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
929 ctxt->ops->put_fpu(ctxt);
932 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
935 ctxt->ops->get_fpu(ctxt);
937 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
938 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
939 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
940 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
941 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
942 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
943 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
944 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
946 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
947 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
948 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
949 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
950 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
951 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
952 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
953 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
957 ctxt->ops->put_fpu(ctxt);
960 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
962 ctxt->ops->get_fpu(ctxt);
964 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
965 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
966 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
967 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
968 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
969 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
970 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
971 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
974 ctxt->ops->put_fpu(ctxt);
977 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
979 ctxt->ops->get_fpu(ctxt);
981 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
982 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
983 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
984 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
985 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
986 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
987 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
988 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
991 ctxt->ops->put_fpu(ctxt);
994 static int em_fninit(struct x86_emulate_ctxt *ctxt)
996 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
997 return emulate_nm(ctxt);
999 ctxt->ops->get_fpu(ctxt);
1000 asm volatile("fninit");
1001 ctxt->ops->put_fpu(ctxt);
1002 return X86EMUL_CONTINUE;
1005 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1009 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1010 return emulate_nm(ctxt);
1012 ctxt->ops->get_fpu(ctxt);
1013 asm volatile("fnstcw %0": "+m"(fcw));
1014 ctxt->ops->put_fpu(ctxt);
1016 /* force 2 byte destination */
1017 ctxt->dst.bytes = 2;
1018 ctxt->dst.val = fcw;
1020 return X86EMUL_CONTINUE;
1023 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1027 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1028 return emulate_nm(ctxt);
1030 ctxt->ops->get_fpu(ctxt);
1031 asm volatile("fnstsw %0": "+m"(fsw));
1032 ctxt->ops->put_fpu(ctxt);
1034 /* force 2 byte destination */
1035 ctxt->dst.bytes = 2;
1036 ctxt->dst.val = fsw;
1038 return X86EMUL_CONTINUE;
1041 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1044 unsigned reg = ctxt->modrm_reg;
1046 if (!(ctxt->d & ModRM))
1047 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1049 if (ctxt->d & Sse) {
1053 read_sse_reg(ctxt, &op->vec_val, reg);
1056 if (ctxt->d & Mmx) {
1065 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1066 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1068 fetch_register_operand(op);
1069 op->orig_val = op->val;
1072 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1074 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1075 ctxt->modrm_seg = VCPU_SREG_SS;
1078 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1082 int index_reg = 0, base_reg = 0, scale;
1083 int rc = X86EMUL_CONTINUE;
1086 if (ctxt->rex_prefix) {
1087 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1088 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1089 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1092 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1093 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1094 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1095 ctxt->modrm_seg = VCPU_SREG_DS;
1097 if (ctxt->modrm_mod == 3) {
1099 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1100 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1102 if (ctxt->d & Sse) {
1105 op->addr.xmm = ctxt->modrm_rm;
1106 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1109 if (ctxt->d & Mmx) {
1112 op->addr.xmm = ctxt->modrm_rm & 7;
1115 fetch_register_operand(op);
1121 if (ctxt->ad_bytes == 2) {
1122 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1123 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1124 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1125 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1127 /* 16-bit ModR/M decode. */
1128 switch (ctxt->modrm_mod) {
1130 if (ctxt->modrm_rm == 6)
1131 modrm_ea += insn_fetch(u16, ctxt);
1134 modrm_ea += insn_fetch(s8, ctxt);
1137 modrm_ea += insn_fetch(u16, ctxt);
1140 switch (ctxt->modrm_rm) {
1142 modrm_ea += bx + si;
1145 modrm_ea += bx + di;
1148 modrm_ea += bp + si;
1151 modrm_ea += bp + di;
1160 if (ctxt->modrm_mod != 0)
1167 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1168 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1169 ctxt->modrm_seg = VCPU_SREG_SS;
1170 modrm_ea = (u16)modrm_ea;
1172 /* 32/64-bit ModR/M decode. */
1173 if ((ctxt->modrm_rm & 7) == 4) {
1174 sib = insn_fetch(u8, ctxt);
1175 index_reg |= (sib >> 3) & 7;
1176 base_reg |= sib & 7;
1179 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1180 modrm_ea += insn_fetch(s32, ctxt);
1182 modrm_ea += reg_read(ctxt, base_reg);
1183 adjust_modrm_seg(ctxt, base_reg);
1186 modrm_ea += reg_read(ctxt, index_reg) << scale;
1187 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1188 if (ctxt->mode == X86EMUL_MODE_PROT64)
1189 ctxt->rip_relative = 1;
1191 base_reg = ctxt->modrm_rm;
1192 modrm_ea += reg_read(ctxt, base_reg);
1193 adjust_modrm_seg(ctxt, base_reg);
1195 switch (ctxt->modrm_mod) {
1197 if (ctxt->modrm_rm == 5)
1198 modrm_ea += insn_fetch(s32, ctxt);
1201 modrm_ea += insn_fetch(s8, ctxt);
1204 modrm_ea += insn_fetch(s32, ctxt);
1208 op->addr.mem.ea = modrm_ea;
1213 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1216 int rc = X86EMUL_CONTINUE;
1219 switch (ctxt->ad_bytes) {
1221 op->addr.mem.ea = insn_fetch(u16, ctxt);
1224 op->addr.mem.ea = insn_fetch(u32, ctxt);
1227 op->addr.mem.ea = insn_fetch(u64, ctxt);
1234 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1238 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1239 mask = ~(ctxt->dst.bytes * 8 - 1);
1241 if (ctxt->src.bytes == 2)
1242 sv = (s16)ctxt->src.val & (s16)mask;
1243 else if (ctxt->src.bytes == 4)
1244 sv = (s32)ctxt->src.val & (s32)mask;
1246 ctxt->dst.addr.mem.ea += (sv >> 3);
1249 /* only subword offset */
1250 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1253 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1254 unsigned long addr, void *dest, unsigned size)
1257 struct read_cache *mc = &ctxt->mem_read;
1259 if (mc->pos < mc->end)
1262 WARN_ON((mc->end + size) >= sizeof(mc->data));
1264 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1266 if (rc != X86EMUL_CONTINUE)
1272 memcpy(dest, mc->data + mc->pos, size);
1274 return X86EMUL_CONTINUE;
1277 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1278 struct segmented_address addr,
1285 rc = linearize(ctxt, addr, size, false, &linear);
1286 if (rc != X86EMUL_CONTINUE)
1288 return read_emulated(ctxt, linear, data, size);
1291 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1292 struct segmented_address addr,
1299 rc = linearize(ctxt, addr, size, true, &linear);
1300 if (rc != X86EMUL_CONTINUE)
1302 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1306 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1307 struct segmented_address addr,
1308 const void *orig_data, const void *data,
1314 rc = linearize(ctxt, addr, size, true, &linear);
1315 if (rc != X86EMUL_CONTINUE)
1317 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1318 size, &ctxt->exception);
1321 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1322 unsigned int size, unsigned short port,
1325 struct read_cache *rc = &ctxt->io_read;
1327 if (rc->pos == rc->end) { /* refill pio read ahead */
1328 unsigned int in_page, n;
1329 unsigned int count = ctxt->rep_prefix ?
1330 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1331 in_page = (ctxt->eflags & EFLG_DF) ?
1332 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1333 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1334 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1338 rc->pos = rc->end = 0;
1339 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1344 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1345 ctxt->dst.data = rc->data + rc->pos;
1346 ctxt->dst.type = OP_MEM_STR;
1347 ctxt->dst.count = (rc->end - rc->pos) / size;
1350 memcpy(dest, rc->data + rc->pos, size);
1356 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1357 u16 index, struct desc_struct *desc)
1362 ctxt->ops->get_idt(ctxt, &dt);
1364 if (dt.size < index * 8 + 7)
1365 return emulate_gp(ctxt, index << 3 | 0x2);
1367 addr = dt.address + index * 8;
1368 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1372 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1373 u16 selector, struct desc_ptr *dt)
1375 const struct x86_emulate_ops *ops = ctxt->ops;
1377 if (selector & 1 << 2) {
1378 struct desc_struct desc;
1381 memset (dt, 0, sizeof *dt);
1382 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1385 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1386 dt->address = get_desc_base(&desc);
1388 ops->get_gdt(ctxt, dt);
1391 /* allowed just for 8 bytes segments */
1392 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1393 u16 selector, struct desc_struct *desc,
1397 u16 index = selector >> 3;
1400 get_descriptor_table_ptr(ctxt, selector, &dt);
1402 if (dt.size < index * 8 + 7)
1403 return emulate_gp(ctxt, selector & 0xfffc);
1405 *desc_addr_p = addr = dt.address + index * 8;
1406 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1410 /* allowed just for 8 bytes segments */
1411 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1412 u16 selector, struct desc_struct *desc)
1415 u16 index = selector >> 3;
1418 get_descriptor_table_ptr(ctxt, selector, &dt);
1420 if (dt.size < index * 8 + 7)
1421 return emulate_gp(ctxt, selector & 0xfffc);
1423 addr = dt.address + index * 8;
1424 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1428 /* Does not support long mode */
1429 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1430 u16 selector, int seg)
1432 struct desc_struct seg_desc, old_desc;
1434 unsigned err_vec = GP_VECTOR;
1436 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1441 memset(&seg_desc, 0, sizeof seg_desc);
1443 if (ctxt->mode == X86EMUL_MODE_REAL) {
1444 /* set real mode segment descriptor (keep limit etc. for
1446 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1447 set_desc_base(&seg_desc, selector << 4);
1449 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1450 /* VM86 needs a clean new segment descriptor */
1451 set_desc_base(&seg_desc, selector << 4);
1452 set_desc_limit(&seg_desc, 0xffff);
1461 cpl = ctxt->ops->cpl(ctxt);
1463 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1464 if ((seg == VCPU_SREG_CS
1465 || (seg == VCPU_SREG_SS
1466 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1467 || seg == VCPU_SREG_TR)
1471 /* TR should be in GDT only */
1472 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1475 if (null_selector) /* for NULL selector skip all following checks */
1478 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1479 if (ret != X86EMUL_CONTINUE)
1482 err_code = selector & 0xfffc;
1483 err_vec = GP_VECTOR;
1485 /* can't load system descriptor into segment selector */
1486 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1490 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1499 * segment is not a writable data segment or segment
1500 * selector's RPL != CPL or segment selector's RPL != CPL
1502 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1506 if (!(seg_desc.type & 8))
1509 if (seg_desc.type & 4) {
1515 if (rpl > cpl || dpl != cpl)
1518 /* CS(RPL) <- CPL */
1519 selector = (selector & 0xfffc) | cpl;
1522 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1524 old_desc = seg_desc;
1525 seg_desc.type |= 2; /* busy */
1526 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1527 sizeof(seg_desc), &ctxt->exception);
1528 if (ret != X86EMUL_CONTINUE)
1531 case VCPU_SREG_LDTR:
1532 if (seg_desc.s || seg_desc.type != 2)
1535 default: /* DS, ES, FS, or GS */
1537 * segment is not a data or readable code segment or
1538 * ((segment is a data or nonconforming code segment)
1539 * and (both RPL and CPL > DPL))
1541 if ((seg_desc.type & 0xa) == 0x8 ||
1542 (((seg_desc.type & 0xc) != 0xc) &&
1543 (rpl > dpl && cpl > dpl)))
1549 /* mark segment as accessed */
1551 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1552 if (ret != X86EMUL_CONTINUE)
1556 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1557 return X86EMUL_CONTINUE;
1559 emulate_exception(ctxt, err_vec, err_code, true);
1560 return X86EMUL_PROPAGATE_FAULT;
1563 static void write_register_operand(struct operand *op)
1565 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1566 switch (op->bytes) {
1568 *(u8 *)op->addr.reg = (u8)op->val;
1571 *(u16 *)op->addr.reg = (u16)op->val;
1574 *op->addr.reg = (u32)op->val;
1575 break; /* 64b: zero-extend */
1577 *op->addr.reg = op->val;
1582 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1588 write_register_operand(op);
1591 if (ctxt->lock_prefix)
1592 rc = segmented_cmpxchg(ctxt,
1598 rc = segmented_write(ctxt,
1602 if (rc != X86EMUL_CONTINUE)
1606 rc = segmented_write(ctxt,
1609 op->bytes * op->count);
1610 if (rc != X86EMUL_CONTINUE)
1614 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1617 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1625 return X86EMUL_CONTINUE;
1628 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1630 struct segmented_address addr;
1632 rsp_increment(ctxt, -bytes);
1633 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1634 addr.seg = VCPU_SREG_SS;
1636 return segmented_write(ctxt, addr, data, bytes);
1639 static int em_push(struct x86_emulate_ctxt *ctxt)
1641 /* Disable writeback. */
1642 ctxt->dst.type = OP_NONE;
1643 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1646 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1647 void *dest, int len)
1650 struct segmented_address addr;
1652 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1653 addr.seg = VCPU_SREG_SS;
1654 rc = segmented_read(ctxt, addr, dest, len);
1655 if (rc != X86EMUL_CONTINUE)
1658 rsp_increment(ctxt, len);
1662 static int em_pop(struct x86_emulate_ctxt *ctxt)
1664 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1667 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1668 void *dest, int len)
1671 unsigned long val, change_mask;
1672 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1673 int cpl = ctxt->ops->cpl(ctxt);
1675 rc = emulate_pop(ctxt, &val, len);
1676 if (rc != X86EMUL_CONTINUE)
1679 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1680 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1682 switch(ctxt->mode) {
1683 case X86EMUL_MODE_PROT64:
1684 case X86EMUL_MODE_PROT32:
1685 case X86EMUL_MODE_PROT16:
1687 change_mask |= EFLG_IOPL;
1689 change_mask |= EFLG_IF;
1691 case X86EMUL_MODE_VM86:
1693 return emulate_gp(ctxt, 0);
1694 change_mask |= EFLG_IF;
1696 default: /* real mode */
1697 change_mask |= (EFLG_IOPL | EFLG_IF);
1701 *(unsigned long *)dest =
1702 (ctxt->eflags & ~change_mask) | (val & change_mask);
1707 static int em_popf(struct x86_emulate_ctxt *ctxt)
1709 ctxt->dst.type = OP_REG;
1710 ctxt->dst.addr.reg = &ctxt->eflags;
1711 ctxt->dst.bytes = ctxt->op_bytes;
1712 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1715 static int em_enter(struct x86_emulate_ctxt *ctxt)
1718 unsigned frame_size = ctxt->src.val;
1719 unsigned nesting_level = ctxt->src2.val & 31;
1723 return X86EMUL_UNHANDLEABLE;
1725 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1726 rc = push(ctxt, &rbp, stack_size(ctxt));
1727 if (rc != X86EMUL_CONTINUE)
1729 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1731 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1732 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1734 return X86EMUL_CONTINUE;
1737 static int em_leave(struct x86_emulate_ctxt *ctxt)
1739 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1741 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1744 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1746 int seg = ctxt->src2.val;
1748 ctxt->src.val = get_segment_selector(ctxt, seg);
1750 return em_push(ctxt);
1753 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1755 int seg = ctxt->src2.val;
1756 unsigned long selector;
1759 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1760 if (rc != X86EMUL_CONTINUE)
1763 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1767 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1769 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1770 int rc = X86EMUL_CONTINUE;
1771 int reg = VCPU_REGS_RAX;
1773 while (reg <= VCPU_REGS_RDI) {
1774 (reg == VCPU_REGS_RSP) ?
1775 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1778 if (rc != X86EMUL_CONTINUE)
1787 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1789 ctxt->src.val = (unsigned long)ctxt->eflags;
1790 return em_push(ctxt);
1793 static int em_popa(struct x86_emulate_ctxt *ctxt)
1795 int rc = X86EMUL_CONTINUE;
1796 int reg = VCPU_REGS_RDI;
1798 while (reg >= VCPU_REGS_RAX) {
1799 if (reg == VCPU_REGS_RSP) {
1800 rsp_increment(ctxt, ctxt->op_bytes);
1804 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1812 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1814 const struct x86_emulate_ops *ops = ctxt->ops;
1821 /* TODO: Add limit checks */
1822 ctxt->src.val = ctxt->eflags;
1824 if (rc != X86EMUL_CONTINUE)
1827 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1829 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1831 if (rc != X86EMUL_CONTINUE)
1834 ctxt->src.val = ctxt->_eip;
1836 if (rc != X86EMUL_CONTINUE)
1839 ops->get_idt(ctxt, &dt);
1841 eip_addr = dt.address + (irq << 2);
1842 cs_addr = dt.address + (irq << 2) + 2;
1844 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1845 if (rc != X86EMUL_CONTINUE)
1848 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1849 if (rc != X86EMUL_CONTINUE)
1852 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1853 if (rc != X86EMUL_CONTINUE)
1861 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1865 invalidate_registers(ctxt);
1866 rc = __emulate_int_real(ctxt, irq);
1867 if (rc == X86EMUL_CONTINUE)
1868 writeback_registers(ctxt);
1872 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1874 switch(ctxt->mode) {
1875 case X86EMUL_MODE_REAL:
1876 return __emulate_int_real(ctxt, irq);
1877 case X86EMUL_MODE_VM86:
1878 case X86EMUL_MODE_PROT16:
1879 case X86EMUL_MODE_PROT32:
1880 case X86EMUL_MODE_PROT64:
1882 /* Protected mode interrupts unimplemented yet */
1883 return X86EMUL_UNHANDLEABLE;
1887 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1889 int rc = X86EMUL_CONTINUE;
1890 unsigned long temp_eip = 0;
1891 unsigned long temp_eflags = 0;
1892 unsigned long cs = 0;
1893 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1894 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1895 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1896 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1898 /* TODO: Add stack limit check */
1900 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1902 if (rc != X86EMUL_CONTINUE)
1905 if (temp_eip & ~0xffff)
1906 return emulate_gp(ctxt, 0);
1908 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1910 if (rc != X86EMUL_CONTINUE)
1913 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1915 if (rc != X86EMUL_CONTINUE)
1918 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1920 if (rc != X86EMUL_CONTINUE)
1923 ctxt->_eip = temp_eip;
1926 if (ctxt->op_bytes == 4)
1927 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1928 else if (ctxt->op_bytes == 2) {
1929 ctxt->eflags &= ~0xffff;
1930 ctxt->eflags |= temp_eflags;
1933 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1934 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1939 static int em_iret(struct x86_emulate_ctxt *ctxt)
1941 switch(ctxt->mode) {
1942 case X86EMUL_MODE_REAL:
1943 return emulate_iret_real(ctxt);
1944 case X86EMUL_MODE_VM86:
1945 case X86EMUL_MODE_PROT16:
1946 case X86EMUL_MODE_PROT32:
1947 case X86EMUL_MODE_PROT64:
1949 /* iret from protected mode unimplemented yet */
1950 return X86EMUL_UNHANDLEABLE;
1954 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1959 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1961 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1962 if (rc != X86EMUL_CONTINUE)
1966 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1967 return X86EMUL_CONTINUE;
1970 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1972 int rc = X86EMUL_CONTINUE;
1974 switch (ctxt->modrm_reg) {
1975 case 2: /* call near abs */ {
1977 old_eip = ctxt->_eip;
1978 ctxt->_eip = ctxt->src.val;
1979 ctxt->src.val = old_eip;
1983 case 4: /* jmp abs */
1984 ctxt->_eip = ctxt->src.val;
1986 case 5: /* jmp far */
1987 rc = em_jmp_far(ctxt);
1996 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1998 u64 old = ctxt->dst.orig_val64;
2000 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2001 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2002 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2003 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2004 ctxt->eflags &= ~EFLG_ZF;
2006 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2007 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2009 ctxt->eflags |= EFLG_ZF;
2011 return X86EMUL_CONTINUE;
2014 static int em_ret(struct x86_emulate_ctxt *ctxt)
2016 ctxt->dst.type = OP_REG;
2017 ctxt->dst.addr.reg = &ctxt->_eip;
2018 ctxt->dst.bytes = ctxt->op_bytes;
2019 return em_pop(ctxt);
2022 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2026 int cpl = ctxt->ops->cpl(ctxt);
2028 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2029 if (rc != X86EMUL_CONTINUE)
2031 if (ctxt->op_bytes == 4)
2032 ctxt->_eip = (u32)ctxt->_eip;
2033 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2034 if (rc != X86EMUL_CONTINUE)
2036 /* Outer-privilege level return is not implemented */
2037 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2038 return X86EMUL_UNHANDLEABLE;
2039 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2043 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2047 rc = em_ret_far(ctxt);
2048 if (rc != X86EMUL_CONTINUE)
2050 rsp_increment(ctxt, ctxt->src.val);
2051 return X86EMUL_CONTINUE;
2054 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2056 /* Save real source value, then compare EAX against destination. */
2057 ctxt->src.orig_val = ctxt->src.val;
2058 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2059 fastop(ctxt, em_cmp);
2061 if (ctxt->eflags & EFLG_ZF) {
2062 /* Success: write back to memory. */
2063 ctxt->dst.val = ctxt->src.orig_val;
2065 /* Failure: write the value we saw to EAX. */
2066 ctxt->dst.type = OP_REG;
2067 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2069 return X86EMUL_CONTINUE;
2072 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2074 int seg = ctxt->src2.val;
2078 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2080 rc = load_segment_descriptor(ctxt, sel, seg);
2081 if (rc != X86EMUL_CONTINUE)
2084 ctxt->dst.val = ctxt->src.val;
2089 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2090 struct desc_struct *cs, struct desc_struct *ss)
2092 cs->l = 0; /* will be adjusted later */
2093 set_desc_base(cs, 0); /* flat segment */
2094 cs->g = 1; /* 4kb granularity */
2095 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2096 cs->type = 0x0b; /* Read, Execute, Accessed */
2098 cs->dpl = 0; /* will be adjusted later */
2103 set_desc_base(ss, 0); /* flat segment */
2104 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2105 ss->g = 1; /* 4kb granularity */
2107 ss->type = 0x03; /* Read/Write, Accessed */
2108 ss->d = 1; /* 32bit stack segment */
2115 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2117 u32 eax, ebx, ecx, edx;
2120 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2121 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2122 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2123 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2126 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2128 const struct x86_emulate_ops *ops = ctxt->ops;
2129 u32 eax, ebx, ecx, edx;
2132 * syscall should always be enabled in longmode - so only become
2133 * vendor specific (cpuid) if other modes are active...
2135 if (ctxt->mode == X86EMUL_MODE_PROT64)
2140 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2142 * Intel ("GenuineIntel")
2143 * remark: Intel CPUs only support "syscall" in 64bit
2144 * longmode. Also an 64bit guest with a
2145 * 32bit compat-app running will #UD !! While this
2146 * behaviour can be fixed (by emulating) into AMD
2147 * response - CPUs of AMD can't behave like Intel.
2149 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2150 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2151 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2154 /* AMD ("AuthenticAMD") */
2155 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2156 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2157 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2160 /* AMD ("AMDisbetter!") */
2161 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2162 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2163 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2166 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2170 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2172 const struct x86_emulate_ops *ops = ctxt->ops;
2173 struct desc_struct cs, ss;
2178 /* syscall is not available in real mode */
2179 if (ctxt->mode == X86EMUL_MODE_REAL ||
2180 ctxt->mode == X86EMUL_MODE_VM86)
2181 return emulate_ud(ctxt);
2183 if (!(em_syscall_is_enabled(ctxt)))
2184 return emulate_ud(ctxt);
2186 ops->get_msr(ctxt, MSR_EFER, &efer);
2187 setup_syscalls_segments(ctxt, &cs, &ss);
2189 if (!(efer & EFER_SCE))
2190 return emulate_ud(ctxt);
2192 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2194 cs_sel = (u16)(msr_data & 0xfffc);
2195 ss_sel = (u16)(msr_data + 8);
2197 if (efer & EFER_LMA) {
2201 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2202 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2204 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2205 if (efer & EFER_LMA) {
2206 #ifdef CONFIG_X86_64
2207 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2210 ctxt->mode == X86EMUL_MODE_PROT64 ?
2211 MSR_LSTAR : MSR_CSTAR, &msr_data);
2212 ctxt->_eip = msr_data;
2214 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2215 ctxt->eflags &= ~(msr_data | EFLG_RF);
2219 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2220 ctxt->_eip = (u32)msr_data;
2222 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2225 return X86EMUL_CONTINUE;
2228 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2230 const struct x86_emulate_ops *ops = ctxt->ops;
2231 struct desc_struct cs, ss;
2236 ops->get_msr(ctxt, MSR_EFER, &efer);
2237 /* inject #GP if in real mode */
2238 if (ctxt->mode == X86EMUL_MODE_REAL)
2239 return emulate_gp(ctxt, 0);
2242 * Not recognized on AMD in compat mode (but is recognized in legacy
2245 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2246 && !vendor_intel(ctxt))
2247 return emulate_ud(ctxt);
2249 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2250 * Therefore, we inject an #UD.
2252 if (ctxt->mode == X86EMUL_MODE_PROT64)
2253 return emulate_ud(ctxt);
2255 setup_syscalls_segments(ctxt, &cs, &ss);
2257 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2258 switch (ctxt->mode) {
2259 case X86EMUL_MODE_PROT32:
2260 if ((msr_data & 0xfffc) == 0x0)
2261 return emulate_gp(ctxt, 0);
2263 case X86EMUL_MODE_PROT64:
2264 if (msr_data == 0x0)
2265 return emulate_gp(ctxt, 0);
2271 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2272 cs_sel = (u16)msr_data;
2273 cs_sel &= ~SELECTOR_RPL_MASK;
2274 ss_sel = cs_sel + 8;
2275 ss_sel &= ~SELECTOR_RPL_MASK;
2276 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2281 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2282 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2284 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2285 ctxt->_eip = msr_data;
2287 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2288 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2290 return X86EMUL_CONTINUE;
2293 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2295 const struct x86_emulate_ops *ops = ctxt->ops;
2296 struct desc_struct cs, ss;
2299 u16 cs_sel = 0, ss_sel = 0;
2301 /* inject #GP if in real mode or Virtual 8086 mode */
2302 if (ctxt->mode == X86EMUL_MODE_REAL ||
2303 ctxt->mode == X86EMUL_MODE_VM86)
2304 return emulate_gp(ctxt, 0);
2306 setup_syscalls_segments(ctxt, &cs, &ss);
2308 if ((ctxt->rex_prefix & 0x8) != 0x0)
2309 usermode = X86EMUL_MODE_PROT64;
2311 usermode = X86EMUL_MODE_PROT32;
2315 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2317 case X86EMUL_MODE_PROT32:
2318 cs_sel = (u16)(msr_data + 16);
2319 if ((msr_data & 0xfffc) == 0x0)
2320 return emulate_gp(ctxt, 0);
2321 ss_sel = (u16)(msr_data + 24);
2323 case X86EMUL_MODE_PROT64:
2324 cs_sel = (u16)(msr_data + 32);
2325 if (msr_data == 0x0)
2326 return emulate_gp(ctxt, 0);
2327 ss_sel = cs_sel + 8;
2332 cs_sel |= SELECTOR_RPL_MASK;
2333 ss_sel |= SELECTOR_RPL_MASK;
2335 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2336 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2338 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2339 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2341 return X86EMUL_CONTINUE;
2344 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2347 if (ctxt->mode == X86EMUL_MODE_REAL)
2349 if (ctxt->mode == X86EMUL_MODE_VM86)
2351 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2352 return ctxt->ops->cpl(ctxt) > iopl;
2355 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2358 const struct x86_emulate_ops *ops = ctxt->ops;
2359 struct desc_struct tr_seg;
2362 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2363 unsigned mask = (1 << len) - 1;
2366 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2369 if (desc_limit_scaled(&tr_seg) < 103)
2371 base = get_desc_base(&tr_seg);
2372 #ifdef CONFIG_X86_64
2373 base |= ((u64)base3) << 32;
2375 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2376 if (r != X86EMUL_CONTINUE)
2378 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2380 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2381 if (r != X86EMUL_CONTINUE)
2383 if ((perm >> bit_idx) & mask)
2388 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2394 if (emulator_bad_iopl(ctxt))
2395 if (!emulator_io_port_access_allowed(ctxt, port, len))
2398 ctxt->perm_ok = true;
2403 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2404 struct tss_segment_16 *tss)
2406 tss->ip = ctxt->_eip;
2407 tss->flag = ctxt->eflags;
2408 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2409 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2410 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2411 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2412 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2413 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2414 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2415 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2417 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2418 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2419 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2420 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2421 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2424 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2425 struct tss_segment_16 *tss)
2429 ctxt->_eip = tss->ip;
2430 ctxt->eflags = tss->flag | 2;
2431 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2432 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2433 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2434 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2435 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2436 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2437 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2438 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2441 * SDM says that segment selectors are loaded before segment
2444 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2445 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2446 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2447 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2448 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2451 * Now load segment descriptors. If fault happens at this stage
2452 * it is handled in a context of new task
2454 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2455 if (ret != X86EMUL_CONTINUE)
2457 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2458 if (ret != X86EMUL_CONTINUE)
2460 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2461 if (ret != X86EMUL_CONTINUE)
2463 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2464 if (ret != X86EMUL_CONTINUE)
2466 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2467 if (ret != X86EMUL_CONTINUE)
2470 return X86EMUL_CONTINUE;
2473 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2474 u16 tss_selector, u16 old_tss_sel,
2475 ulong old_tss_base, struct desc_struct *new_desc)
2477 const struct x86_emulate_ops *ops = ctxt->ops;
2478 struct tss_segment_16 tss_seg;
2480 u32 new_tss_base = get_desc_base(new_desc);
2482 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2484 if (ret != X86EMUL_CONTINUE)
2485 /* FIXME: need to provide precise fault address */
2488 save_state_to_tss16(ctxt, &tss_seg);
2490 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2492 if (ret != X86EMUL_CONTINUE)
2493 /* FIXME: need to provide precise fault address */
2496 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2498 if (ret != X86EMUL_CONTINUE)
2499 /* FIXME: need to provide precise fault address */
2502 if (old_tss_sel != 0xffff) {
2503 tss_seg.prev_task_link = old_tss_sel;
2505 ret = ops->write_std(ctxt, new_tss_base,
2506 &tss_seg.prev_task_link,
2507 sizeof tss_seg.prev_task_link,
2509 if (ret != X86EMUL_CONTINUE)
2510 /* FIXME: need to provide precise fault address */
2514 return load_state_from_tss16(ctxt, &tss_seg);
2517 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2518 struct tss_segment_32 *tss)
2520 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2521 tss->eip = ctxt->_eip;
2522 tss->eflags = ctxt->eflags;
2523 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2524 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2525 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2526 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2527 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2528 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2529 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2530 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2532 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2533 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2534 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2535 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2536 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2537 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2538 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2541 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2542 struct tss_segment_32 *tss)
2546 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2547 return emulate_gp(ctxt, 0);
2548 ctxt->_eip = tss->eip;
2549 ctxt->eflags = tss->eflags | 2;
2551 /* General purpose registers */
2552 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2553 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2554 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2555 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2556 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2557 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2558 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2559 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2562 * SDM says that segment selectors are loaded before segment
2565 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2566 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2567 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2568 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2569 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2570 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2571 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2574 * If we're switching between Protected Mode and VM86, we need to make
2575 * sure to update the mode before loading the segment descriptors so
2576 * that the selectors are interpreted correctly.
2578 * Need to get rflags to the vcpu struct immediately because it
2579 * influences the CPL which is checked at least when loading the segment
2580 * descriptors and when pushing an error code to the new kernel stack.
2582 * TODO Introduce a separate ctxt->ops->set_cpl callback
2584 if (ctxt->eflags & X86_EFLAGS_VM)
2585 ctxt->mode = X86EMUL_MODE_VM86;
2587 ctxt->mode = X86EMUL_MODE_PROT32;
2589 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2592 * Now load segment descriptors. If fault happenes at this stage
2593 * it is handled in a context of new task
2595 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2596 if (ret != X86EMUL_CONTINUE)
2598 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2599 if (ret != X86EMUL_CONTINUE)
2601 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2602 if (ret != X86EMUL_CONTINUE)
2604 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2605 if (ret != X86EMUL_CONTINUE)
2607 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2608 if (ret != X86EMUL_CONTINUE)
2610 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2611 if (ret != X86EMUL_CONTINUE)
2613 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2614 if (ret != X86EMUL_CONTINUE)
2617 return X86EMUL_CONTINUE;
2620 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2621 u16 tss_selector, u16 old_tss_sel,
2622 ulong old_tss_base, struct desc_struct *new_desc)
2624 const struct x86_emulate_ops *ops = ctxt->ops;
2625 struct tss_segment_32 tss_seg;
2627 u32 new_tss_base = get_desc_base(new_desc);
2629 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2631 if (ret != X86EMUL_CONTINUE)
2632 /* FIXME: need to provide precise fault address */
2635 save_state_to_tss32(ctxt, &tss_seg);
2637 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2639 if (ret != X86EMUL_CONTINUE)
2640 /* FIXME: need to provide precise fault address */
2643 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2645 if (ret != X86EMUL_CONTINUE)
2646 /* FIXME: need to provide precise fault address */
2649 if (old_tss_sel != 0xffff) {
2650 tss_seg.prev_task_link = old_tss_sel;
2652 ret = ops->write_std(ctxt, new_tss_base,
2653 &tss_seg.prev_task_link,
2654 sizeof tss_seg.prev_task_link,
2656 if (ret != X86EMUL_CONTINUE)
2657 /* FIXME: need to provide precise fault address */
2661 return load_state_from_tss32(ctxt, &tss_seg);
2664 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2665 u16 tss_selector, int idt_index, int reason,
2666 bool has_error_code, u32 error_code)
2668 const struct x86_emulate_ops *ops = ctxt->ops;
2669 struct desc_struct curr_tss_desc, next_tss_desc;
2671 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2672 ulong old_tss_base =
2673 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2677 /* FIXME: old_tss_base == ~0 ? */
2679 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2680 if (ret != X86EMUL_CONTINUE)
2682 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2683 if (ret != X86EMUL_CONTINUE)
2686 /* FIXME: check that next_tss_desc is tss */
2689 * Check privileges. The three cases are task switch caused by...
2691 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2692 * 2. Exception/IRQ/iret: No check is performed
2693 * 3. jmp/call to TSS: Check against DPL of the TSS
2695 if (reason == TASK_SWITCH_GATE) {
2696 if (idt_index != -1) {
2697 /* Software interrupts */
2698 struct desc_struct task_gate_desc;
2701 ret = read_interrupt_descriptor(ctxt, idt_index,
2703 if (ret != X86EMUL_CONTINUE)
2706 dpl = task_gate_desc.dpl;
2707 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2708 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2710 } else if (reason != TASK_SWITCH_IRET) {
2711 int dpl = next_tss_desc.dpl;
2712 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2713 return emulate_gp(ctxt, tss_selector);
2717 desc_limit = desc_limit_scaled(&next_tss_desc);
2718 if (!next_tss_desc.p ||
2719 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2720 desc_limit < 0x2b)) {
2721 emulate_ts(ctxt, tss_selector & 0xfffc);
2722 return X86EMUL_PROPAGATE_FAULT;
2725 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2726 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2727 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2730 if (reason == TASK_SWITCH_IRET)
2731 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2733 /* set back link to prev task only if NT bit is set in eflags
2734 note that old_tss_sel is not used after this point */
2735 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2736 old_tss_sel = 0xffff;
2738 if (next_tss_desc.type & 8)
2739 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2740 old_tss_base, &next_tss_desc);
2742 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2743 old_tss_base, &next_tss_desc);
2744 if (ret != X86EMUL_CONTINUE)
2747 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2748 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2750 if (reason != TASK_SWITCH_IRET) {
2751 next_tss_desc.type |= (1 << 1); /* set busy flag */
2752 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2755 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2756 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2758 if (has_error_code) {
2759 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2760 ctxt->lock_prefix = 0;
2761 ctxt->src.val = (unsigned long) error_code;
2762 ret = em_push(ctxt);
2768 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2769 u16 tss_selector, int idt_index, int reason,
2770 bool has_error_code, u32 error_code)
2774 invalidate_registers(ctxt);
2775 ctxt->_eip = ctxt->eip;
2776 ctxt->dst.type = OP_NONE;
2778 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2779 has_error_code, error_code);
2781 if (rc == X86EMUL_CONTINUE) {
2782 ctxt->eip = ctxt->_eip;
2783 writeback_registers(ctxt);
2786 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2789 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2792 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2794 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2795 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2798 static int em_das(struct x86_emulate_ctxt *ctxt)
2801 bool af, cf, old_cf;
2803 cf = ctxt->eflags & X86_EFLAGS_CF;
2809 af = ctxt->eflags & X86_EFLAGS_AF;
2810 if ((al & 0x0f) > 9 || af) {
2812 cf = old_cf | (al >= 250);
2817 if (old_al > 0x99 || old_cf) {
2823 /* Set PF, ZF, SF */
2824 ctxt->src.type = OP_IMM;
2826 ctxt->src.bytes = 1;
2827 fastop(ctxt, em_or);
2828 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2830 ctxt->eflags |= X86_EFLAGS_CF;
2832 ctxt->eflags |= X86_EFLAGS_AF;
2833 return X86EMUL_CONTINUE;
2836 static int em_aam(struct x86_emulate_ctxt *ctxt)
2840 if (ctxt->src.val == 0)
2841 return emulate_de(ctxt);
2843 al = ctxt->dst.val & 0xff;
2844 ah = al / ctxt->src.val;
2845 al %= ctxt->src.val;
2847 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2849 /* Set PF, ZF, SF */
2850 ctxt->src.type = OP_IMM;
2852 ctxt->src.bytes = 1;
2853 fastop(ctxt, em_or);
2855 return X86EMUL_CONTINUE;
2858 static int em_aad(struct x86_emulate_ctxt *ctxt)
2860 u8 al = ctxt->dst.val & 0xff;
2861 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2863 al = (al + (ah * ctxt->src.val)) & 0xff;
2865 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2867 /* Set PF, ZF, SF */
2868 ctxt->src.type = OP_IMM;
2870 ctxt->src.bytes = 1;
2871 fastop(ctxt, em_or);
2873 return X86EMUL_CONTINUE;
2876 static int em_call(struct x86_emulate_ctxt *ctxt)
2878 long rel = ctxt->src.val;
2880 ctxt->src.val = (unsigned long)ctxt->_eip;
2882 return em_push(ctxt);
2885 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2891 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2892 old_eip = ctxt->_eip;
2894 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2895 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2896 return X86EMUL_CONTINUE;
2899 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2901 ctxt->src.val = old_cs;
2903 if (rc != X86EMUL_CONTINUE)
2906 ctxt->src.val = old_eip;
2907 return em_push(ctxt);
2910 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2914 ctxt->dst.type = OP_REG;
2915 ctxt->dst.addr.reg = &ctxt->_eip;
2916 ctxt->dst.bytes = ctxt->op_bytes;
2917 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2918 if (rc != X86EMUL_CONTINUE)
2920 rsp_increment(ctxt, ctxt->src.val);
2921 return X86EMUL_CONTINUE;
2924 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2926 /* Write back the register source. */
2927 ctxt->src.val = ctxt->dst.val;
2928 write_register_operand(&ctxt->src);
2930 /* Write back the memory destination with implicit LOCK prefix. */
2931 ctxt->dst.val = ctxt->src.orig_val;
2932 ctxt->lock_prefix = 1;
2933 return X86EMUL_CONTINUE;
2936 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2938 ctxt->dst.val = ctxt->src2.val;
2939 return fastop(ctxt, em_imul);
2942 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2944 ctxt->dst.type = OP_REG;
2945 ctxt->dst.bytes = ctxt->src.bytes;
2946 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2947 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2949 return X86EMUL_CONTINUE;
2952 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2956 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2957 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2958 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2959 return X86EMUL_CONTINUE;
2962 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2966 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2967 return emulate_gp(ctxt, 0);
2968 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2969 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2970 return X86EMUL_CONTINUE;
2973 static int em_mov(struct x86_emulate_ctxt *ctxt)
2975 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2976 return X86EMUL_CONTINUE;
2979 #define FFL(x) bit(X86_FEATURE_##x)
2981 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2983 u32 ebx, ecx, edx, eax = 1;
2987 * Check MOVBE is set in the guest-visible CPUID leaf.
2989 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2990 if (!(ecx & FFL(MOVBE)))
2991 return emulate_ud(ctxt);
2993 switch (ctxt->op_bytes) {
2996 * From MOVBE definition: "...When the operand size is 16 bits,
2997 * the upper word of the destination register remains unchanged
3000 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3001 * rules so we have to do the operation almost per hand.
3003 tmp = (u16)ctxt->src.val;
3004 ctxt->dst.val &= ~0xffffUL;
3005 ctxt->dst.val |= (unsigned long)swab16(tmp);
3008 ctxt->dst.val = swab32((u32)ctxt->src.val);
3011 ctxt->dst.val = swab64(ctxt->src.val);
3014 return X86EMUL_PROPAGATE_FAULT;
3016 return X86EMUL_CONTINUE;
3019 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3021 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3022 return emulate_gp(ctxt, 0);
3024 /* Disable writeback. */
3025 ctxt->dst.type = OP_NONE;
3026 return X86EMUL_CONTINUE;
3029 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3033 if (ctxt->mode == X86EMUL_MODE_PROT64)
3034 val = ctxt->src.val & ~0ULL;
3036 val = ctxt->src.val & ~0U;
3038 /* #UD condition is already handled. */
3039 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3040 return emulate_gp(ctxt, 0);
3042 /* Disable writeback. */
3043 ctxt->dst.type = OP_NONE;
3044 return X86EMUL_CONTINUE;
3047 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3051 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3052 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3053 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3054 return emulate_gp(ctxt, 0);
3056 return X86EMUL_CONTINUE;
3059 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3063 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3064 return emulate_gp(ctxt, 0);
3066 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3067 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3068 return X86EMUL_CONTINUE;
3071 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3073 if (ctxt->modrm_reg > VCPU_SREG_GS)
3074 return emulate_ud(ctxt);
3076 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3077 return X86EMUL_CONTINUE;
3080 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3082 u16 sel = ctxt->src.val;
3084 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3085 return emulate_ud(ctxt);
3087 if (ctxt->modrm_reg == VCPU_SREG_SS)
3088 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3090 /* Disable writeback. */
3091 ctxt->dst.type = OP_NONE;
3092 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3095 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3097 u16 sel = ctxt->src.val;
3099 /* Disable writeback. */
3100 ctxt->dst.type = OP_NONE;
3101 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3104 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3106 u16 sel = ctxt->src.val;
3108 /* Disable writeback. */
3109 ctxt->dst.type = OP_NONE;
3110 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3113 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3118 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3119 if (rc == X86EMUL_CONTINUE)
3120 ctxt->ops->invlpg(ctxt, linear);
3121 /* Disable writeback. */
3122 ctxt->dst.type = OP_NONE;
3123 return X86EMUL_CONTINUE;
3126 static int em_clts(struct x86_emulate_ctxt *ctxt)
3130 cr0 = ctxt->ops->get_cr(ctxt, 0);
3132 ctxt->ops->set_cr(ctxt, 0, cr0);
3133 return X86EMUL_CONTINUE;
3136 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3140 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3141 return X86EMUL_UNHANDLEABLE;
3143 rc = ctxt->ops->fix_hypercall(ctxt);
3144 if (rc != X86EMUL_CONTINUE)
3147 /* Let the processor re-execute the fixed hypercall */
3148 ctxt->_eip = ctxt->eip;
3149 /* Disable writeback. */
3150 ctxt->dst.type = OP_NONE;
3151 return X86EMUL_CONTINUE;
3154 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3155 void (*get)(struct x86_emulate_ctxt *ctxt,
3156 struct desc_ptr *ptr))
3158 struct desc_ptr desc_ptr;
3160 if (ctxt->mode == X86EMUL_MODE_PROT64)
3162 get(ctxt, &desc_ptr);
3163 if (ctxt->op_bytes == 2) {
3165 desc_ptr.address &= 0x00ffffff;
3167 /* Disable writeback. */
3168 ctxt->dst.type = OP_NONE;
3169 return segmented_write(ctxt, ctxt->dst.addr.mem,
3170 &desc_ptr, 2 + ctxt->op_bytes);
3173 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3175 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3178 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3180 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3183 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3185 struct desc_ptr desc_ptr;
3188 if (ctxt->mode == X86EMUL_MODE_PROT64)
3190 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3191 &desc_ptr.size, &desc_ptr.address,
3193 if (rc != X86EMUL_CONTINUE)
3195 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3196 /* Disable writeback. */
3197 ctxt->dst.type = OP_NONE;
3198 return X86EMUL_CONTINUE;
3201 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3205 rc = ctxt->ops->fix_hypercall(ctxt);
3207 /* Disable writeback. */
3208 ctxt->dst.type = OP_NONE;
3212 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3214 struct desc_ptr desc_ptr;
3217 if (ctxt->mode == X86EMUL_MODE_PROT64)
3219 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3220 &desc_ptr.size, &desc_ptr.address,
3222 if (rc != X86EMUL_CONTINUE)
3224 ctxt->ops->set_idt(ctxt, &desc_ptr);
3225 /* Disable writeback. */
3226 ctxt->dst.type = OP_NONE;
3227 return X86EMUL_CONTINUE;
3230 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3232 ctxt->dst.bytes = 2;
3233 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3234 return X86EMUL_CONTINUE;
3237 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3239 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3240 | (ctxt->src.val & 0x0f));
3241 ctxt->dst.type = OP_NONE;
3242 return X86EMUL_CONTINUE;
3245 static int em_loop(struct x86_emulate_ctxt *ctxt)
3247 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3248 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3249 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3250 jmp_rel(ctxt, ctxt->src.val);
3252 return X86EMUL_CONTINUE;
3255 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3257 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3258 jmp_rel(ctxt, ctxt->src.val);
3260 return X86EMUL_CONTINUE;
3263 static int em_in(struct x86_emulate_ctxt *ctxt)
3265 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3267 return X86EMUL_IO_NEEDED;
3269 return X86EMUL_CONTINUE;
3272 static int em_out(struct x86_emulate_ctxt *ctxt)
3274 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3276 /* Disable writeback. */
3277 ctxt->dst.type = OP_NONE;
3278 return X86EMUL_CONTINUE;
3281 static int em_cli(struct x86_emulate_ctxt *ctxt)
3283 if (emulator_bad_iopl(ctxt))
3284 return emulate_gp(ctxt, 0);
3286 ctxt->eflags &= ~X86_EFLAGS_IF;
3287 return X86EMUL_CONTINUE;
3290 static int em_sti(struct x86_emulate_ctxt *ctxt)
3292 if (emulator_bad_iopl(ctxt))
3293 return emulate_gp(ctxt, 0);
3295 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3296 ctxt->eflags |= X86_EFLAGS_IF;
3297 return X86EMUL_CONTINUE;
3300 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3302 u32 eax, ebx, ecx, edx;
3304 eax = reg_read(ctxt, VCPU_REGS_RAX);
3305 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3306 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3307 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3308 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3309 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3310 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3311 return X86EMUL_CONTINUE;
3314 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3318 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3319 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3321 ctxt->eflags &= ~0xffUL;
3322 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3323 return X86EMUL_CONTINUE;
3326 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3328 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3329 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3330 return X86EMUL_CONTINUE;
3333 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3335 switch (ctxt->op_bytes) {
3336 #ifdef CONFIG_X86_64
3338 asm("bswap %0" : "+r"(ctxt->dst.val));
3342 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3345 return X86EMUL_CONTINUE;
3348 static bool valid_cr(int nr)
3360 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3362 if (!valid_cr(ctxt->modrm_reg))
3363 return emulate_ud(ctxt);
3365 return X86EMUL_CONTINUE;
3368 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3370 u64 new_val = ctxt->src.val64;
3371 int cr = ctxt->modrm_reg;
3374 static u64 cr_reserved_bits[] = {
3375 0xffffffff00000000ULL,
3376 0, 0, 0, /* CR3 checked later */
3383 return emulate_ud(ctxt);
3385 if (new_val & cr_reserved_bits[cr])
3386 return emulate_gp(ctxt, 0);
3391 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3392 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3393 return emulate_gp(ctxt, 0);
3395 cr4 = ctxt->ops->get_cr(ctxt, 4);
3396 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3398 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3399 !(cr4 & X86_CR4_PAE))
3400 return emulate_gp(ctxt, 0);
3407 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3408 if (efer & EFER_LMA)
3409 rsvd = CR3_L_MODE_RESERVED_BITS;
3410 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3411 rsvd = CR3_PAE_RESERVED_BITS;
3412 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3413 rsvd = CR3_NONPAE_RESERVED_BITS;
3416 return emulate_gp(ctxt, 0);
3421 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3423 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3424 return emulate_gp(ctxt, 0);
3430 return X86EMUL_CONTINUE;
3433 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3437 ctxt->ops->get_dr(ctxt, 7, &dr7);
3439 /* Check if DR7.Global_Enable is set */
3440 return dr7 & (1 << 13);
3443 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3445 int dr = ctxt->modrm_reg;
3449 return emulate_ud(ctxt);
3451 cr4 = ctxt->ops->get_cr(ctxt, 4);
3452 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3453 return emulate_ud(ctxt);
3455 if (check_dr7_gd(ctxt))
3456 return emulate_db(ctxt);
3458 return X86EMUL_CONTINUE;
3461 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3463 u64 new_val = ctxt->src.val64;
3464 int dr = ctxt->modrm_reg;
3466 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3467 return emulate_gp(ctxt, 0);
3469 return check_dr_read(ctxt);
3472 static int check_svme(struct x86_emulate_ctxt *ctxt)
3476 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3478 if (!(efer & EFER_SVME))
3479 return emulate_ud(ctxt);
3481 return X86EMUL_CONTINUE;
3484 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3486 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3488 /* Valid physical address? */
3489 if (rax & 0xffff000000000000ULL)
3490 return emulate_gp(ctxt, 0);
3492 return check_svme(ctxt);
3495 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3497 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3499 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3500 return emulate_ud(ctxt);
3502 return X86EMUL_CONTINUE;
3505 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3507 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3508 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3510 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3512 return emulate_gp(ctxt, 0);
3514 return X86EMUL_CONTINUE;
3517 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3519 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3520 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3521 return emulate_gp(ctxt, 0);
3523 return X86EMUL_CONTINUE;
3526 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3528 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3529 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3530 return emulate_gp(ctxt, 0);
3532 return X86EMUL_CONTINUE;
3535 #define D(_y) { .flags = (_y) }
3536 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3537 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3538 .check_perm = (_p) }
3539 #define N D(NotImpl)
3540 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3541 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3542 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3543 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3544 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3545 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3546 #define II(_f, _e, _i) \
3547 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3548 #define IIP(_f, _e, _i, _p) \
3549 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3550 .check_perm = (_p) }
3551 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3553 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3554 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3555 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3556 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3557 #define I2bvIP(_f, _e, _i, _p) \
3558 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3560 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3561 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3562 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3564 static const struct opcode group7_rm1[] = {
3565 DI(SrcNone | Priv, monitor),
3566 DI(SrcNone | Priv, mwait),
3570 static const struct opcode group7_rm3[] = {
3571 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3572 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3573 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3574 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3575 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3576 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3577 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3578 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3581 static const struct opcode group7_rm7[] = {
3583 DIP(SrcNone, rdtscp, check_rdtsc),
3587 static const struct opcode group1[] = {
3589 F(Lock | PageTable, em_or),
3592 F(Lock | PageTable, em_and),
3598 static const struct opcode group1A[] = {
3599 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3602 static const struct opcode group2[] = {
3603 F(DstMem | ModRM, em_rol),
3604 F(DstMem | ModRM, em_ror),
3605 F(DstMem | ModRM, em_rcl),
3606 F(DstMem | ModRM, em_rcr),
3607 F(DstMem | ModRM, em_shl),
3608 F(DstMem | ModRM, em_shr),
3609 F(DstMem | ModRM, em_shl),
3610 F(DstMem | ModRM, em_sar),
3613 static const struct opcode group3[] = {
3614 F(DstMem | SrcImm | NoWrite, em_test),
3615 F(DstMem | SrcImm | NoWrite, em_test),
3616 F(DstMem | SrcNone | Lock, em_not),
3617 F(DstMem | SrcNone | Lock, em_neg),
3618 F(DstXacc | Src2Mem, em_mul_ex),
3619 F(DstXacc | Src2Mem, em_imul_ex),
3620 F(DstXacc | Src2Mem, em_div_ex),
3621 F(DstXacc | Src2Mem, em_idiv_ex),
3624 static const struct opcode group4[] = {
3625 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3626 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3630 static const struct opcode group5[] = {
3631 F(DstMem | SrcNone | Lock, em_inc),
3632 F(DstMem | SrcNone | Lock, em_dec),
3633 I(SrcMem | Stack, em_grp45),
3634 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3635 I(SrcMem | Stack, em_grp45),
3636 I(SrcMemFAddr | ImplicitOps, em_grp45),
3637 I(SrcMem | Stack, em_grp45), D(Undefined),
3640 static const struct opcode group6[] = {
3643 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3644 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3648 static const struct group_dual group7 = { {
3649 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3650 II(Mov | DstMem | Priv, em_sidt, sidt),
3651 II(SrcMem | Priv, em_lgdt, lgdt),
3652 II(SrcMem | Priv, em_lidt, lidt),
3653 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3654 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3655 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3657 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3659 N, EXT(0, group7_rm3),
3660 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3661 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3665 static const struct opcode group8[] = {
3667 F(DstMem | SrcImmByte | NoWrite, em_bt),
3668 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3669 F(DstMem | SrcImmByte | Lock, em_btr),
3670 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3673 static const struct group_dual group9 = { {
3674 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3676 N, N, N, N, N, N, N, N,
3679 static const struct opcode group11[] = {
3680 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3684 static const struct gprefix pfx_0f_6f_0f_7f = {
3685 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3688 static const struct gprefix pfx_vmovntpx = {
3689 I(0, em_mov), N, N, N,
3692 static const struct escape escape_d9 = { {
3693 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3696 N, N, N, N, N, N, N, N,
3698 N, N, N, N, N, N, N, N,
3700 N, N, N, N, N, N, N, N,
3702 N, N, N, N, N, N, N, N,
3704 N, N, N, N, N, N, N, N,
3706 N, N, N, N, N, N, N, N,
3708 N, N, N, N, N, N, N, N,
3710 N, N, N, N, N, N, N, N,
3713 static const struct escape escape_db = { {
3714 N, N, N, N, N, N, N, N,
3717 N, N, N, N, N, N, N, N,
3719 N, N, N, N, N, N, N, N,
3721 N, N, N, N, N, N, N, N,
3723 N, N, N, N, N, N, N, N,
3725 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3727 N, N, N, N, N, N, N, N,
3729 N, N, N, N, N, N, N, N,
3731 N, N, N, N, N, N, N, N,
3734 static const struct escape escape_dd = { {
3735 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3738 N, N, N, N, N, N, N, N,
3740 N, N, N, N, N, N, N, N,
3742 N, N, N, N, N, N, N, N,
3744 N, N, N, N, N, N, N, N,
3746 N, N, N, N, N, N, N, N,
3748 N, N, N, N, N, N, N, N,
3750 N, N, N, N, N, N, N, N,
3752 N, N, N, N, N, N, N, N,
3755 static const struct opcode opcode_table[256] = {
3757 F6ALU(Lock, em_add),
3758 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3759 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3761 F6ALU(Lock | PageTable, em_or),
3762 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3765 F6ALU(Lock, em_adc),
3766 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3767 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3769 F6ALU(Lock, em_sbb),
3770 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3771 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3773 F6ALU(Lock | PageTable, em_and), N, N,
3775 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3777 F6ALU(Lock, em_xor), N, N,
3779 F6ALU(NoWrite, em_cmp), N, N,
3781 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3783 X8(I(SrcReg | Stack, em_push)),
3785 X8(I(DstReg | Stack, em_pop)),
3787 I(ImplicitOps | Stack | No64, em_pusha),
3788 I(ImplicitOps | Stack | No64, em_popa),
3789 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3792 I(SrcImm | Mov | Stack, em_push),
3793 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3794 I(SrcImmByte | Mov | Stack, em_push),
3795 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3796 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3797 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3801 G(ByteOp | DstMem | SrcImm, group1),
3802 G(DstMem | SrcImm, group1),
3803 G(ByteOp | DstMem | SrcImm | No64, group1),
3804 G(DstMem | SrcImmByte, group1),
3805 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3806 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3808 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3809 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3810 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3811 D(ModRM | SrcMem | NoAccess | DstReg),
3812 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3815 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3817 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3818 I(SrcImmFAddr | No64, em_call_far), N,
3819 II(ImplicitOps | Stack, em_pushf, pushf),
3820 II(ImplicitOps | Stack, em_popf, popf),
3821 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3823 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3824 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3825 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3826 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3828 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3829 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3830 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3831 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3833 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3835 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3837 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3838 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3839 I(ImplicitOps | Stack, em_ret),
3840 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3841 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3842 G(ByteOp, group11), G(0, group11),
3844 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3845 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3846 I(ImplicitOps | Stack, em_ret_far),
3847 D(ImplicitOps), DI(SrcImmByte, intn),
3848 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3850 G(Src2One | ByteOp, group2), G(Src2One, group2),
3851 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3852 I(DstAcc | SrcImmUByte | No64, em_aam),
3853 I(DstAcc | SrcImmUByte | No64, em_aad),
3854 F(DstAcc | ByteOp | No64, em_salc),
3855 I(DstAcc | SrcXLat | ByteOp, em_mov),
3857 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3859 X3(I(SrcImmByte, em_loop)),
3860 I(SrcImmByte, em_jcxz),
3861 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3862 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3864 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3865 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3866 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3867 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3869 N, DI(ImplicitOps, icebp), N, N,
3870 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3871 G(ByteOp, group3), G(0, group3),
3873 D(ImplicitOps), D(ImplicitOps),
3874 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3875 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3878 static const struct opcode twobyte_table[256] = {
3880 G(0, group6), GD(0, &group7), N, N,
3881 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3882 II(ImplicitOps | Priv, em_clts, clts), N,
3883 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3884 N, D(ImplicitOps | ModRM), N, N,
3886 N, N, N, N, N, N, N, N,
3887 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3889 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3890 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3891 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3892 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3894 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3897 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3898 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3899 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3900 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3901 I(ImplicitOps | EmulateOnUD, em_sysenter),
3902 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3904 N, N, N, N, N, N, N, N,
3906 X16(D(DstReg | SrcMem | ModRM | Mov)),
3908 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3913 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3918 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3922 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3924 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3925 II(ImplicitOps, em_cpuid, cpuid),
3926 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3927 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3928 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3930 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3931 DI(ImplicitOps, rsm),
3932 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3933 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3934 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3935 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3937 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3938 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3939 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3940 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3941 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3942 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3946 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3947 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3948 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3950 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3951 N, D(DstMem | SrcReg | ModRM | Mov),
3952 N, N, N, GD(0, &group9),
3954 X8(I(DstReg, em_bswap)),
3956 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3958 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3960 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3963 static const struct gprefix three_byte_0f_38_f0 = {
3964 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3967 static const struct gprefix three_byte_0f_38_f1 = {
3968 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3972 * Insns below are selected by the prefix which indexed by the third opcode
3975 static const struct opcode opcode_map_0f_38[256] = {
3977 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3979 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3981 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3982 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4001 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4005 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4011 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4012 unsigned size, bool sign_extension)
4014 int rc = X86EMUL_CONTINUE;
4018 op->addr.mem.ea = ctxt->_eip;
4019 /* NB. Immediates are sign-extended as necessary. */
4020 switch (op->bytes) {
4022 op->val = insn_fetch(s8, ctxt);
4025 op->val = insn_fetch(s16, ctxt);
4028 op->val = insn_fetch(s32, ctxt);
4031 op->val = insn_fetch(s64, ctxt);
4034 if (!sign_extension) {
4035 switch (op->bytes) {
4043 op->val &= 0xffffffff;
4051 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4054 int rc = X86EMUL_CONTINUE;
4058 decode_register_operand(ctxt, op);
4061 rc = decode_imm(ctxt, op, 1, false);
4064 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4068 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4069 fetch_bit_operand(ctxt);
4070 op->orig_val = op->val;
4073 ctxt->memop.bytes = 8;
4077 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4078 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4079 fetch_register_operand(op);
4080 op->orig_val = op->val;
4084 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4085 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4086 fetch_register_operand(op);
4087 op->orig_val = op->val;
4090 if (ctxt->d & ByteOp) {
4095 op->bytes = ctxt->op_bytes;
4096 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4097 fetch_register_operand(op);
4098 op->orig_val = op->val;
4102 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4104 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4105 op->addr.mem.seg = VCPU_SREG_ES;
4112 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4113 fetch_register_operand(op);
4117 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4120 rc = decode_imm(ctxt, op, 1, true);
4127 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4130 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4133 ctxt->memop.bytes = 1;
4134 if (ctxt->memop.type == OP_REG) {
4135 ctxt->memop.addr.reg = decode_register(ctxt,
4136 ctxt->modrm_rm, true);
4137 fetch_register_operand(&ctxt->memop);
4141 ctxt->memop.bytes = 2;
4144 ctxt->memop.bytes = 4;
4147 rc = decode_imm(ctxt, op, 2, false);
4150 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4154 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4156 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4157 op->addr.mem.seg = seg_override(ctxt);
4163 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4165 register_address(ctxt,
4166 reg_read(ctxt, VCPU_REGS_RBX) +
4167 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4168 op->addr.mem.seg = seg_override(ctxt);
4173 op->addr.mem.ea = ctxt->_eip;
4174 op->bytes = ctxt->op_bytes + 2;
4175 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4178 ctxt->memop.bytes = ctxt->op_bytes + 2;
4181 op->val = VCPU_SREG_ES;
4184 op->val = VCPU_SREG_CS;
4187 op->val = VCPU_SREG_SS;
4190 op->val = VCPU_SREG_DS;
4193 op->val = VCPU_SREG_FS;
4196 op->val = VCPU_SREG_GS;
4199 /* Special instructions do their own operand decoding. */
4201 op->type = OP_NONE; /* Disable writeback. */
4209 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4211 int rc = X86EMUL_CONTINUE;
4212 int mode = ctxt->mode;
4213 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4214 bool op_prefix = false;
4215 struct opcode opcode;
4217 ctxt->memop.type = OP_NONE;
4218 ctxt->memopp = NULL;
4219 ctxt->_eip = ctxt->eip;
4220 ctxt->fetch.start = ctxt->_eip;
4221 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4222 ctxt->opcode_len = 1;
4224 memcpy(ctxt->fetch.data, insn, insn_len);
4227 case X86EMUL_MODE_REAL:
4228 case X86EMUL_MODE_VM86:
4229 case X86EMUL_MODE_PROT16:
4230 def_op_bytes = def_ad_bytes = 2;
4232 case X86EMUL_MODE_PROT32:
4233 def_op_bytes = def_ad_bytes = 4;
4235 #ifdef CONFIG_X86_64
4236 case X86EMUL_MODE_PROT64:
4242 return EMULATION_FAILED;
4245 ctxt->op_bytes = def_op_bytes;
4246 ctxt->ad_bytes = def_ad_bytes;
4248 /* Legacy prefixes. */
4250 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4251 case 0x66: /* operand-size override */
4253 /* switch between 2/4 bytes */
4254 ctxt->op_bytes = def_op_bytes ^ 6;
4256 case 0x67: /* address-size override */
4257 if (mode == X86EMUL_MODE_PROT64)
4258 /* switch between 4/8 bytes */
4259 ctxt->ad_bytes = def_ad_bytes ^ 12;
4261 /* switch between 2/4 bytes */
4262 ctxt->ad_bytes = def_ad_bytes ^ 6;
4264 case 0x26: /* ES override */
4265 case 0x2e: /* CS override */
4266 case 0x36: /* SS override */
4267 case 0x3e: /* DS override */
4268 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4270 case 0x64: /* FS override */
4271 case 0x65: /* GS override */
4272 set_seg_override(ctxt, ctxt->b & 7);
4274 case 0x40 ... 0x4f: /* REX */
4275 if (mode != X86EMUL_MODE_PROT64)
4277 ctxt->rex_prefix = ctxt->b;
4279 case 0xf0: /* LOCK */
4280 ctxt->lock_prefix = 1;
4282 case 0xf2: /* REPNE/REPNZ */
4283 case 0xf3: /* REP/REPE/REPZ */
4284 ctxt->rep_prefix = ctxt->b;
4290 /* Any legacy prefix after a REX prefix nullifies its effect. */
4292 ctxt->rex_prefix = 0;
4298 if (ctxt->rex_prefix & 8)
4299 ctxt->op_bytes = 8; /* REX.W */
4301 /* Opcode byte(s). */
4302 opcode = opcode_table[ctxt->b];
4303 /* Two-byte opcode? */
4304 if (ctxt->b == 0x0f) {
4305 ctxt->opcode_len = 2;
4306 ctxt->b = insn_fetch(u8, ctxt);
4307 opcode = twobyte_table[ctxt->b];
4309 /* 0F_38 opcode map */
4310 if (ctxt->b == 0x38) {
4311 ctxt->opcode_len = 3;
4312 ctxt->b = insn_fetch(u8, ctxt);
4313 opcode = opcode_map_0f_38[ctxt->b];
4316 ctxt->d = opcode.flags;
4318 if (ctxt->d & ModRM)
4319 ctxt->modrm = insn_fetch(u8, ctxt);
4321 while (ctxt->d & GroupMask) {
4322 switch (ctxt->d & GroupMask) {
4324 goffset = (ctxt->modrm >> 3) & 7;
4325 opcode = opcode.u.group[goffset];
4328 goffset = (ctxt->modrm >> 3) & 7;
4329 if ((ctxt->modrm >> 6) == 3)
4330 opcode = opcode.u.gdual->mod3[goffset];
4332 opcode = opcode.u.gdual->mod012[goffset];
4335 goffset = ctxt->modrm & 7;
4336 opcode = opcode.u.group[goffset];
4339 if (ctxt->rep_prefix && op_prefix)
4340 return EMULATION_FAILED;
4341 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4342 switch (simd_prefix) {
4343 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4344 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4345 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4346 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4350 if (ctxt->modrm > 0xbf)
4351 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4353 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4356 return EMULATION_FAILED;
4359 ctxt->d &= ~(u64)GroupMask;
4360 ctxt->d |= opcode.flags;
4363 ctxt->execute = opcode.u.execute;
4364 ctxt->check_perm = opcode.check_perm;
4365 ctxt->intercept = opcode.intercept;
4368 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4369 return EMULATION_FAILED;
4371 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4372 return EMULATION_FAILED;
4374 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4377 if (ctxt->d & Op3264) {
4378 if (mode == X86EMUL_MODE_PROT64)
4385 ctxt->op_bytes = 16;
4386 else if (ctxt->d & Mmx)
4389 /* ModRM and SIB bytes. */
4390 if (ctxt->d & ModRM) {
4391 rc = decode_modrm(ctxt, &ctxt->memop);
4392 if (!ctxt->has_seg_override)
4393 set_seg_override(ctxt, ctxt->modrm_seg);
4394 } else if (ctxt->d & MemAbs)
4395 rc = decode_abs(ctxt, &ctxt->memop);
4396 if (rc != X86EMUL_CONTINUE)
4399 if (!ctxt->has_seg_override)
4400 set_seg_override(ctxt, VCPU_SREG_DS);
4402 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4404 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4405 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4408 * Decode and fetch the source operand: register, memory
4411 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4412 if (rc != X86EMUL_CONTINUE)
4416 * Decode and fetch the second source operand: register, memory
4419 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4420 if (rc != X86EMUL_CONTINUE)
4423 /* Decode and fetch the destination operand: register or memory. */
4424 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4427 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4428 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4430 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4433 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4435 return ctxt->d & PageTable;
4438 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4440 /* The second termination condition only applies for REPE
4441 * and REPNE. Test if the repeat string operation prefix is
4442 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4443 * corresponding termination condition according to:
4444 * - if REPE/REPZ and ZF = 0 then done
4445 * - if REPNE/REPNZ and ZF = 1 then done
4447 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4448 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4449 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4450 ((ctxt->eflags & EFLG_ZF) == 0))
4451 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4452 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4458 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4462 ctxt->ops->get_fpu(ctxt);
4463 asm volatile("1: fwait \n\t"
4465 ".pushsection .fixup,\"ax\" \n\t"
4467 "movb $1, %[fault] \n\t"
4470 _ASM_EXTABLE(1b, 3b)
4471 : [fault]"+qm"(fault));
4472 ctxt->ops->put_fpu(ctxt);
4474 if (unlikely(fault))
4475 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4477 return X86EMUL_CONTINUE;
4480 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4483 if (op->type == OP_MM)
4484 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4487 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4489 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4490 if (!(ctxt->d & ByteOp))
4491 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4492 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4493 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4495 : "c"(ctxt->src2.val));
4496 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4497 if (!fop) /* exception is returned in fop variable */
4498 return emulate_de(ctxt);
4499 return X86EMUL_CONTINUE;
4502 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4504 const struct x86_emulate_ops *ops = ctxt->ops;
4505 int rc = X86EMUL_CONTINUE;
4506 int saved_dst_type = ctxt->dst.type;
4508 ctxt->mem_read.pos = 0;
4510 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4511 (ctxt->d & Undefined)) {
4512 rc = emulate_ud(ctxt);
4516 /* LOCK prefix is allowed only with some instructions */
4517 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4518 rc = emulate_ud(ctxt);
4522 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4523 rc = emulate_ud(ctxt);
4527 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4528 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4529 rc = emulate_ud(ctxt);
4533 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4534 rc = emulate_nm(ctxt);
4538 if (ctxt->d & Mmx) {
4539 rc = flush_pending_x87_faults(ctxt);
4540 if (rc != X86EMUL_CONTINUE)
4543 * Now that we know the fpu is exception safe, we can fetch
4546 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4547 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4548 if (!(ctxt->d & Mov))
4549 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4552 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4553 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4554 X86_ICPT_PRE_EXCEPT);
4555 if (rc != X86EMUL_CONTINUE)
4559 /* Privileged instruction can be executed only in CPL=0 */
4560 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4561 rc = emulate_gp(ctxt, 0);
4565 /* Instruction can only be executed in protected mode */
4566 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4567 rc = emulate_ud(ctxt);
4571 /* Do instruction specific permission checks */
4572 if (ctxt->check_perm) {
4573 rc = ctxt->check_perm(ctxt);
4574 if (rc != X86EMUL_CONTINUE)
4578 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4579 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4580 X86_ICPT_POST_EXCEPT);
4581 if (rc != X86EMUL_CONTINUE)
4585 if (ctxt->rep_prefix && (ctxt->d & String)) {
4586 /* All REP prefixes have the same first termination condition */
4587 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4588 ctxt->eip = ctxt->_eip;
4593 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4594 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4595 ctxt->src.valptr, ctxt->src.bytes);
4596 if (rc != X86EMUL_CONTINUE)
4598 ctxt->src.orig_val64 = ctxt->src.val64;
4601 if (ctxt->src2.type == OP_MEM) {
4602 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4603 &ctxt->src2.val, ctxt->src2.bytes);
4604 if (rc != X86EMUL_CONTINUE)
4608 if ((ctxt->d & DstMask) == ImplicitOps)
4612 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4613 /* optimisation - avoid slow emulated read if Mov */
4614 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4615 &ctxt->dst.val, ctxt->dst.bytes);
4616 if (rc != X86EMUL_CONTINUE)
4619 ctxt->dst.orig_val = ctxt->dst.val;
4623 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4624 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4625 X86_ICPT_POST_MEMACCESS);
4626 if (rc != X86EMUL_CONTINUE)
4630 if (ctxt->execute) {
4631 if (ctxt->d & Fastop) {
4632 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4633 rc = fastop(ctxt, fop);
4634 if (rc != X86EMUL_CONTINUE)
4638 rc = ctxt->execute(ctxt);
4639 if (rc != X86EMUL_CONTINUE)
4644 if (ctxt->opcode_len == 2)
4646 else if (ctxt->opcode_len == 3)
4647 goto threebyte_insn;
4650 case 0x63: /* movsxd */
4651 if (ctxt->mode != X86EMUL_MODE_PROT64)
4652 goto cannot_emulate;
4653 ctxt->dst.val = (s32) ctxt->src.val;
4655 case 0x70 ... 0x7f: /* jcc (short) */
4656 if (test_cc(ctxt->b, ctxt->eflags))
4657 jmp_rel(ctxt, ctxt->src.val);
4659 case 0x8d: /* lea r16/r32, m */
4660 ctxt->dst.val = ctxt->src.addr.mem.ea;
4662 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4663 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4667 case 0x98: /* cbw/cwde/cdqe */
4668 switch (ctxt->op_bytes) {
4669 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4670 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4671 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4674 case 0xcc: /* int3 */
4675 rc = emulate_int(ctxt, 3);
4677 case 0xcd: /* int n */
4678 rc = emulate_int(ctxt, ctxt->src.val);
4680 case 0xce: /* into */
4681 if (ctxt->eflags & EFLG_OF)
4682 rc = emulate_int(ctxt, 4);
4684 case 0xe9: /* jmp rel */
4685 case 0xeb: /* jmp rel short */
4686 jmp_rel(ctxt, ctxt->src.val);
4687 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4689 case 0xf4: /* hlt */
4690 ctxt->ops->halt(ctxt);
4692 case 0xf5: /* cmc */
4693 /* complement carry flag from eflags reg */
4694 ctxt->eflags ^= EFLG_CF;
4696 case 0xf8: /* clc */
4697 ctxt->eflags &= ~EFLG_CF;
4699 case 0xf9: /* stc */
4700 ctxt->eflags |= EFLG_CF;
4702 case 0xfc: /* cld */
4703 ctxt->eflags &= ~EFLG_DF;
4705 case 0xfd: /* std */
4706 ctxt->eflags |= EFLG_DF;
4709 goto cannot_emulate;
4712 if (rc != X86EMUL_CONTINUE)
4716 if (!(ctxt->d & NoWrite)) {
4717 rc = writeback(ctxt, &ctxt->dst);
4718 if (rc != X86EMUL_CONTINUE)
4721 if (ctxt->d & SrcWrite) {
4722 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4723 rc = writeback(ctxt, &ctxt->src);
4724 if (rc != X86EMUL_CONTINUE)
4729 * restore dst type in case the decoding will be reused
4730 * (happens for string instruction )
4732 ctxt->dst.type = saved_dst_type;
4734 if ((ctxt->d & SrcMask) == SrcSI)
4735 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4737 if ((ctxt->d & DstMask) == DstDI)
4738 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4740 if (ctxt->rep_prefix && (ctxt->d & String)) {
4742 struct read_cache *r = &ctxt->io_read;
4743 if ((ctxt->d & SrcMask) == SrcSI)
4744 count = ctxt->src.count;
4746 count = ctxt->dst.count;
4747 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4750 if (!string_insn_completed(ctxt)) {
4752 * Re-enter guest when pio read ahead buffer is empty
4753 * or, if it is not used, after each 1024 iteration.
4755 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4756 (r->end == 0 || r->end != r->pos)) {
4758 * Reset read cache. Usually happens before
4759 * decode, but since instruction is restarted
4760 * we have to do it here.
4762 ctxt->mem_read.end = 0;
4763 writeback_registers(ctxt);
4764 return EMULATION_RESTART;
4766 goto done; /* skip rip writeback */
4770 ctxt->eip = ctxt->_eip;
4773 if (rc == X86EMUL_PROPAGATE_FAULT)
4774 ctxt->have_exception = true;
4775 if (rc == X86EMUL_INTERCEPTED)
4776 return EMULATION_INTERCEPTED;
4778 if (rc == X86EMUL_CONTINUE)
4779 writeback_registers(ctxt);
4781 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4785 case 0x09: /* wbinvd */
4786 (ctxt->ops->wbinvd)(ctxt);
4788 case 0x08: /* invd */
4789 case 0x0d: /* GrpP (prefetch) */
4790 case 0x18: /* Grp16 (prefetch/nop) */
4791 case 0x1f: /* nop */
4793 case 0x20: /* mov cr, reg */
4794 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4796 case 0x21: /* mov from dr to reg */
4797 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4799 case 0x40 ... 0x4f: /* cmov */
4800 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4801 if (!test_cc(ctxt->b, ctxt->eflags))
4802 ctxt->dst.type = OP_NONE; /* no writeback */
4804 case 0x80 ... 0x8f: /* jnz rel, etc*/
4805 if (test_cc(ctxt->b, ctxt->eflags))
4806 jmp_rel(ctxt, ctxt->src.val);
4808 case 0x90 ... 0x9f: /* setcc r/m8 */
4809 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4811 case 0xae: /* clflush */
4813 case 0xb6 ... 0xb7: /* movzx */
4814 ctxt->dst.bytes = ctxt->op_bytes;
4815 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4816 : (u16) ctxt->src.val;
4818 case 0xbe ... 0xbf: /* movsx */
4819 ctxt->dst.bytes = ctxt->op_bytes;
4820 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4821 (s16) ctxt->src.val;
4823 case 0xc3: /* movnti */
4824 ctxt->dst.bytes = ctxt->op_bytes;
4825 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4826 (u64) ctxt->src.val;
4829 goto cannot_emulate;
4834 if (rc != X86EMUL_CONTINUE)
4840 return EMULATION_FAILED;
4843 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4845 invalidate_registers(ctxt);
4848 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4850 writeback_registers(ctxt);