1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
145 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
146 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
147 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
148 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
150 #define X2(x...) x, x
151 #define X3(x...) X2(x), x
152 #define X4(x...) X2(x), X2(x)
153 #define X5(x...) X4(x), x
154 #define X6(x...) X4(x), X2(x)
155 #define X7(x...) X4(x), X3(x)
156 #define X8(x...) X4(x), X4(x)
157 #define X16(x...) X8(x), X8(x)
163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 struct opcode *group;
165 struct group_dual *gdual;
166 struct gprefix *gprefix;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
172 struct opcode mod012[8];
173 struct opcode mod3[8];
177 struct opcode pfx_no;
178 struct opcode pfx_66;
179 struct opcode pfx_f2;
180 struct opcode pfx_f3;
183 /* EFLAGS bit definitions. */
184 #define EFLG_ID (1<<21)
185 #define EFLG_VIP (1<<20)
186 #define EFLG_VIF (1<<19)
187 #define EFLG_AC (1<<18)
188 #define EFLG_VM (1<<17)
189 #define EFLG_RF (1<<16)
190 #define EFLG_IOPL (3<<12)
191 #define EFLG_NT (1<<14)
192 #define EFLG_OF (1<<11)
193 #define EFLG_DF (1<<10)
194 #define EFLG_IF (1<<9)
195 #define EFLG_TF (1<<8)
196 #define EFLG_SF (1<<7)
197 #define EFLG_ZF (1<<6)
198 #define EFLG_AF (1<<4)
199 #define EFLG_PF (1<<2)
200 #define EFLG_CF (1<<0)
202 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203 #define EFLG_RESERVED_ONE_MASK 2
206 * Instruction emulation:
207 * Most instructions are emulated directly via a fragment of inline assembly
208 * code. This allows us to save/restore EFLAGS and thus very easily pick up
209 * any modified flags.
212 #if defined(CONFIG_X86_64)
213 #define _LO32 "k" /* force 32-bit operand */
214 #define _STK "%%rsp" /* stack pointer */
215 #elif defined(__i386__)
216 #define _LO32 "" /* force 32-bit operand */
217 #define _STK "%%esp" /* stack pointer */
221 * These EFLAGS bits are restored from saved value during emulation, and
222 * any changes are written back to the saved value after emulation.
224 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
226 /* Before executing instruction: restore necessary bits in EFLAGS. */
227 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
228 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
229 "movl %"_sav",%"_LO32 _tmp"; " \
232 "movl %"_msk",%"_LO32 _tmp"; " \
233 "andl %"_LO32 _tmp",("_STK"); " \
235 "notl %"_LO32 _tmp"; " \
236 "andl %"_LO32 _tmp",("_STK"); " \
237 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
239 "orl %"_LO32 _tmp",("_STK"); " \
243 /* After executing instruction: write-back necessary bits in EFLAGS. */
244 #define _POST_EFLAGS(_sav, _msk, _tmp) \
245 /* _sav |= EFLAGS & _msk; */ \
248 "andl %"_msk",%"_LO32 _tmp"; " \
249 "orl %"_LO32 _tmp",%"_sav"; "
257 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
259 __asm__ __volatile__ ( \
260 _PRE_EFLAGS("0", "4", "2") \
261 _op _suffix " %"_x"3,%1; " \
262 _POST_EFLAGS("0", "4", "2") \
263 : "=m" ((ctxt)->eflags), \
264 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
266 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
270 /* Raw emulation: instruction has two explicit operands. */
271 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
273 unsigned long _tmp; \
275 switch ((ctxt)->dst.bytes) { \
277 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
280 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
283 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
288 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
290 unsigned long _tmp; \
291 switch ((ctxt)->dst.bytes) { \
293 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
296 __emulate_2op_nobyte(ctxt, _op, \
297 _wx, _wy, _lx, _ly, _qx, _qy); \
302 /* Source operand is byte-sized and may be restricted to just %cl. */
303 #define emulate_2op_SrcB(ctxt, _op) \
304 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
306 /* Source operand is byte, word, long or quad sized. */
307 #define emulate_2op_SrcV(ctxt, _op) \
308 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
310 /* Source operand is word, long or quad sized. */
311 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
312 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
314 /* Instruction has three operands and one operand is stored in ECX register */
315 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
317 unsigned long _tmp; \
318 _type _clv = (ctxt)->src2.val; \
319 _type _srcv = (ctxt)->src.val; \
320 _type _dstv = (ctxt)->dst.val; \
322 __asm__ __volatile__ ( \
323 _PRE_EFLAGS("0", "5", "2") \
324 _op _suffix " %4,%1 \n" \
325 _POST_EFLAGS("0", "5", "2") \
326 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
327 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
330 (ctxt)->src2.val = (unsigned long) _clv; \
331 (ctxt)->src2.val = (unsigned long) _srcv; \
332 (ctxt)->dst.val = (unsigned long) _dstv; \
335 #define emulate_2op_cl(ctxt, _op) \
337 switch ((ctxt)->dst.bytes) { \
339 __emulate_2op_cl(ctxt, _op, "w", u16); \
342 __emulate_2op_cl(ctxt, _op, "l", u32); \
345 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
350 #define __emulate_1op(ctxt, _op, _suffix) \
352 unsigned long _tmp; \
354 __asm__ __volatile__ ( \
355 _PRE_EFLAGS("0", "3", "2") \
356 _op _suffix " %1; " \
357 _POST_EFLAGS("0", "3", "2") \
358 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
360 : "i" (EFLAGS_MASK)); \
363 /* Instruction has only one explicit operand (no source operand). */
364 #define emulate_1op(ctxt, _op) \
366 switch ((ctxt)->dst.bytes) { \
367 case 1: __emulate_1op(ctxt, _op, "b"); break; \
368 case 2: __emulate_1op(ctxt, _op, "w"); break; \
369 case 4: __emulate_1op(ctxt, _op, "l"); break; \
370 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
374 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
376 unsigned long _tmp; \
377 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
378 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
380 __asm__ __volatile__ ( \
381 _PRE_EFLAGS("0", "5", "1") \
383 _op _suffix " %6; " \
385 _POST_EFLAGS("0", "5", "1") \
386 ".pushsection .fixup,\"ax\" \n\t" \
387 "3: movb $1, %4 \n\t" \
390 _ASM_EXTABLE(1b, 3b) \
391 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
392 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
393 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
394 "a" (*rax), "d" (*rdx)); \
397 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
398 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
400 switch((ctxt)->src.bytes) { \
402 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
408 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
411 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
416 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
417 enum x86_intercept intercept,
418 enum x86_intercept_stage stage)
420 struct x86_instruction_info info = {
421 .intercept = intercept,
422 .rep_prefix = ctxt->rep_prefix,
423 .modrm_mod = ctxt->modrm_mod,
424 .modrm_reg = ctxt->modrm_reg,
425 .modrm_rm = ctxt->modrm_rm,
426 .src_val = ctxt->src.val64,
427 .src_bytes = ctxt->src.bytes,
428 .dst_bytes = ctxt->dst.bytes,
429 .ad_bytes = ctxt->ad_bytes,
430 .next_rip = ctxt->eip,
433 return ctxt->ops->intercept(ctxt, &info, stage);
436 static void assign_masked(ulong *dest, ulong src, ulong mask)
438 *dest = (*dest & ~mask) | (src & mask);
441 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
443 return (1UL << (ctxt->ad_bytes << 3)) - 1;
446 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
449 struct desc_struct ss;
451 if (ctxt->mode == X86EMUL_MODE_PROT64)
453 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
454 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
457 static int stack_size(struct x86_emulate_ctxt *ctxt)
459 return (__fls(stack_mask(ctxt)) + 1) >> 3;
462 /* Access/update address held in a register, based on addressing mode. */
463 static inline unsigned long
464 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
466 if (ctxt->ad_bytes == sizeof(unsigned long))
469 return reg & ad_mask(ctxt);
472 static inline unsigned long
473 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
475 return address_mask(ctxt, reg);
479 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
481 if (ctxt->ad_bytes == sizeof(unsigned long))
484 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
487 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
489 register_address_increment(ctxt, &ctxt->_eip, rel);
492 static u32 desc_limit_scaled(struct desc_struct *desc)
494 u32 limit = get_desc_limit(desc);
496 return desc->g ? (limit << 12) | 0xfff : limit;
499 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
501 ctxt->has_seg_override = true;
502 ctxt->seg_override = seg;
505 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
507 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
510 return ctxt->ops->get_cached_segment_base(ctxt, seg);
513 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
515 if (!ctxt->has_seg_override)
518 return ctxt->seg_override;
521 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
522 u32 error, bool valid)
524 ctxt->exception.vector = vec;
525 ctxt->exception.error_code = error;
526 ctxt->exception.error_code_valid = valid;
527 return X86EMUL_PROPAGATE_FAULT;
530 static int emulate_db(struct x86_emulate_ctxt *ctxt)
532 return emulate_exception(ctxt, DB_VECTOR, 0, false);
535 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
537 return emulate_exception(ctxt, GP_VECTOR, err, true);
540 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, SS_VECTOR, err, true);
545 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, UD_VECTOR, 0, false);
550 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
552 return emulate_exception(ctxt, TS_VECTOR, err, true);
555 static int emulate_de(struct x86_emulate_ctxt *ctxt)
557 return emulate_exception(ctxt, DE_VECTOR, 0, false);
560 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
562 return emulate_exception(ctxt, NM_VECTOR, 0, false);
565 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
568 struct desc_struct desc;
570 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
574 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
579 struct desc_struct desc;
581 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
582 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
586 * x86 defines three classes of vector instructions: explicitly
587 * aligned, explicitly unaligned, and the rest, which change behaviour
588 * depending on whether they're AVX encoded or not.
590 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
591 * subject to the same check.
593 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
595 if (likely(size < 16))
598 if (ctxt->d & Aligned)
600 else if (ctxt->d & Unaligned)
602 else if (ctxt->d & Avx)
608 static int __linearize(struct x86_emulate_ctxt *ctxt,
609 struct segmented_address addr,
610 unsigned size, bool write, bool fetch,
613 struct desc_struct desc;
620 la = seg_base(ctxt, addr.seg) + addr.ea;
621 switch (ctxt->mode) {
622 case X86EMUL_MODE_REAL:
624 case X86EMUL_MODE_PROT64:
625 if (((signed long)la << 16) >> 16 != la)
626 return emulate_gp(ctxt, 0);
629 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
633 /* code segment or read-only data segment */
634 if (((desc.type & 8) || !(desc.type & 2)) && write)
636 /* unreadable code segment */
637 if (!fetch && (desc.type & 8) && !(desc.type & 2))
639 lim = desc_limit_scaled(&desc);
640 if ((desc.type & 8) || !(desc.type & 4)) {
641 /* expand-up segment */
642 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
645 /* expand-down segment */
646 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
648 lim = desc.d ? 0xffffffff : 0xffff;
649 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
652 cpl = ctxt->ops->cpl(ctxt);
655 if (!(desc.type & 8)) {
659 } else if ((desc.type & 8) && !(desc.type & 4)) {
660 /* nonconforming code segment */
663 } else if ((desc.type & 8) && (desc.type & 4)) {
664 /* conforming code segment */
670 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
672 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
673 return emulate_gp(ctxt, 0);
675 return X86EMUL_CONTINUE;
677 if (addr.seg == VCPU_SREG_SS)
678 return emulate_ss(ctxt, addr.seg);
680 return emulate_gp(ctxt, addr.seg);
683 static int linearize(struct x86_emulate_ctxt *ctxt,
684 struct segmented_address addr,
685 unsigned size, bool write,
688 return __linearize(ctxt, addr, size, write, false, linear);
692 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
693 struct segmented_address addr,
700 rc = linearize(ctxt, addr, size, false, &linear);
701 if (rc != X86EMUL_CONTINUE)
703 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
707 * Fetch the next byte of the instruction being emulated which is pointed to
708 * by ctxt->_eip, then increment ctxt->_eip.
710 * Also prefetch the remaining bytes of the instruction without crossing page
711 * boundary if they are not in fetch_cache yet.
713 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
715 struct fetch_cache *fc = &ctxt->fetch;
719 if (ctxt->_eip == fc->end) {
720 unsigned long linear;
721 struct segmented_address addr = { .seg = VCPU_SREG_CS,
723 cur_size = fc->end - fc->start;
724 size = min(15UL - cur_size,
725 PAGE_SIZE - offset_in_page(ctxt->_eip));
726 rc = __linearize(ctxt, addr, size, false, true, &linear);
727 if (unlikely(rc != X86EMUL_CONTINUE))
729 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
730 size, &ctxt->exception);
731 if (unlikely(rc != X86EMUL_CONTINUE))
735 *dest = fc->data[ctxt->_eip - fc->start];
737 return X86EMUL_CONTINUE;
740 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
741 void *dest, unsigned size)
745 /* x86 instructions are limited to 15 bytes. */
746 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
747 return X86EMUL_UNHANDLEABLE;
749 rc = do_insn_fetch_byte(ctxt, dest++);
750 if (rc != X86EMUL_CONTINUE)
753 return X86EMUL_CONTINUE;
756 /* Fetch next part of the instruction being emulated. */
757 #define insn_fetch(_type, _ctxt) \
758 ({ unsigned long _x; \
759 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
760 if (rc != X86EMUL_CONTINUE) \
765 #define insn_fetch_arr(_arr, _size, _ctxt) \
766 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
767 if (rc != X86EMUL_CONTINUE) \
772 * Given the 'reg' portion of a ModRM byte, and a register block, return a
773 * pointer into the block that addresses the relevant register.
774 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
776 static void *decode_register(u8 modrm_reg, unsigned long *regs,
781 p = ®s[modrm_reg];
782 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
783 p = (unsigned char *)®s[modrm_reg & 3] + 1;
787 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
788 struct segmented_address addr,
789 u16 *size, unsigned long *address, int op_bytes)
796 rc = segmented_read_std(ctxt, addr, size, 2);
797 if (rc != X86EMUL_CONTINUE)
800 rc = segmented_read_std(ctxt, addr, address, op_bytes);
804 static int test_cc(unsigned int condition, unsigned int flags)
808 switch ((condition & 15) >> 1) {
810 rc |= (flags & EFLG_OF);
812 case 1: /* b/c/nae */
813 rc |= (flags & EFLG_CF);
816 rc |= (flags & EFLG_ZF);
819 rc |= (flags & (EFLG_CF|EFLG_ZF));
822 rc |= (flags & EFLG_SF);
825 rc |= (flags & EFLG_PF);
828 rc |= (flags & EFLG_ZF);
831 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
835 /* Odd condition identifiers (lsb == 1) have inverted sense. */
836 return (!!rc ^ (condition & 1));
839 static void fetch_register_operand(struct operand *op)
843 op->val = *(u8 *)op->addr.reg;
846 op->val = *(u16 *)op->addr.reg;
849 op->val = *(u32 *)op->addr.reg;
852 op->val = *(u64 *)op->addr.reg;
857 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
859 ctxt->ops->get_fpu(ctxt);
861 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
862 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
863 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
864 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
865 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
866 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
867 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
868 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
870 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
871 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
872 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
873 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
874 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
875 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
876 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
877 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
881 ctxt->ops->put_fpu(ctxt);
884 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
887 ctxt->ops->get_fpu(ctxt);
889 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
890 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
891 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
892 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
893 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
894 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
895 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
896 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
898 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
899 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
900 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
901 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
902 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
903 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
904 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
905 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
909 ctxt->ops->put_fpu(ctxt);
912 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
914 ctxt->ops->get_fpu(ctxt);
916 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
917 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
918 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
919 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
920 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
921 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
922 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
923 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
926 ctxt->ops->put_fpu(ctxt);
929 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
931 ctxt->ops->get_fpu(ctxt);
933 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
934 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
935 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
936 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
937 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
938 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
939 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
940 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
943 ctxt->ops->put_fpu(ctxt);
946 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
949 unsigned reg = ctxt->modrm_reg;
950 int highbyte_regs = ctxt->rex_prefix == 0;
952 if (!(ctxt->d & ModRM))
953 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
959 read_sse_reg(ctxt, &op->vec_val, reg);
971 if (ctxt->d & ByteOp) {
972 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
975 op->addr.reg = decode_register(reg, ctxt->regs, 0);
976 op->bytes = ctxt->op_bytes;
978 fetch_register_operand(op);
979 op->orig_val = op->val;
982 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
984 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
985 ctxt->modrm_seg = VCPU_SREG_SS;
988 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
992 int index_reg = 0, base_reg = 0, scale;
993 int rc = X86EMUL_CONTINUE;
996 if (ctxt->rex_prefix) {
997 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
998 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
999 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1002 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1003 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1004 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1005 ctxt->modrm_seg = VCPU_SREG_DS;
1007 if (ctxt->modrm_mod == 3) {
1009 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1010 op->addr.reg = decode_register(ctxt->modrm_rm,
1011 ctxt->regs, ctxt->d & ByteOp);
1012 if (ctxt->d & Sse) {
1015 op->addr.xmm = ctxt->modrm_rm;
1016 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1019 if (ctxt->d & Mmx) {
1022 op->addr.xmm = ctxt->modrm_rm & 7;
1025 fetch_register_operand(op);
1031 if (ctxt->ad_bytes == 2) {
1032 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
1033 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
1034 unsigned si = ctxt->regs[VCPU_REGS_RSI];
1035 unsigned di = ctxt->regs[VCPU_REGS_RDI];
1037 /* 16-bit ModR/M decode. */
1038 switch (ctxt->modrm_mod) {
1040 if (ctxt->modrm_rm == 6)
1041 modrm_ea += insn_fetch(u16, ctxt);
1044 modrm_ea += insn_fetch(s8, ctxt);
1047 modrm_ea += insn_fetch(u16, ctxt);
1050 switch (ctxt->modrm_rm) {
1052 modrm_ea += bx + si;
1055 modrm_ea += bx + di;
1058 modrm_ea += bp + si;
1061 modrm_ea += bp + di;
1070 if (ctxt->modrm_mod != 0)
1077 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1078 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1079 ctxt->modrm_seg = VCPU_SREG_SS;
1080 modrm_ea = (u16)modrm_ea;
1082 /* 32/64-bit ModR/M decode. */
1083 if ((ctxt->modrm_rm & 7) == 4) {
1084 sib = insn_fetch(u8, ctxt);
1085 index_reg |= (sib >> 3) & 7;
1086 base_reg |= sib & 7;
1089 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1090 modrm_ea += insn_fetch(s32, ctxt);
1092 modrm_ea += ctxt->regs[base_reg];
1093 adjust_modrm_seg(ctxt, base_reg);
1096 modrm_ea += ctxt->regs[index_reg] << scale;
1097 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1098 if (ctxt->mode == X86EMUL_MODE_PROT64)
1099 ctxt->rip_relative = 1;
1101 base_reg = ctxt->modrm_rm;
1102 modrm_ea += ctxt->regs[base_reg];
1103 adjust_modrm_seg(ctxt, base_reg);
1105 switch (ctxt->modrm_mod) {
1107 if (ctxt->modrm_rm == 5)
1108 modrm_ea += insn_fetch(s32, ctxt);
1111 modrm_ea += insn_fetch(s8, ctxt);
1114 modrm_ea += insn_fetch(s32, ctxt);
1118 op->addr.mem.ea = modrm_ea;
1123 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1126 int rc = X86EMUL_CONTINUE;
1129 switch (ctxt->ad_bytes) {
1131 op->addr.mem.ea = insn_fetch(u16, ctxt);
1134 op->addr.mem.ea = insn_fetch(u32, ctxt);
1137 op->addr.mem.ea = insn_fetch(u64, ctxt);
1144 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1148 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1149 mask = ~(ctxt->dst.bytes * 8 - 1);
1151 if (ctxt->src.bytes == 2)
1152 sv = (s16)ctxt->src.val & (s16)mask;
1153 else if (ctxt->src.bytes == 4)
1154 sv = (s32)ctxt->src.val & (s32)mask;
1156 ctxt->dst.addr.mem.ea += (sv >> 3);
1159 /* only subword offset */
1160 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1163 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1164 unsigned long addr, void *dest, unsigned size)
1167 struct read_cache *mc = &ctxt->mem_read;
1169 if (mc->pos < mc->end)
1172 WARN_ON((mc->end + size) >= sizeof(mc->data));
1174 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1176 if (rc != X86EMUL_CONTINUE)
1182 memcpy(dest, mc->data + mc->pos, size);
1184 return X86EMUL_CONTINUE;
1187 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1188 struct segmented_address addr,
1195 rc = linearize(ctxt, addr, size, false, &linear);
1196 if (rc != X86EMUL_CONTINUE)
1198 return read_emulated(ctxt, linear, data, size);
1201 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1202 struct segmented_address addr,
1209 rc = linearize(ctxt, addr, size, true, &linear);
1210 if (rc != X86EMUL_CONTINUE)
1212 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1216 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1217 struct segmented_address addr,
1218 const void *orig_data, const void *data,
1224 rc = linearize(ctxt, addr, size, true, &linear);
1225 if (rc != X86EMUL_CONTINUE)
1227 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1228 size, &ctxt->exception);
1231 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1232 unsigned int size, unsigned short port,
1235 struct read_cache *rc = &ctxt->io_read;
1237 if (rc->pos == rc->end) { /* refill pio read ahead */
1238 unsigned int in_page, n;
1239 unsigned int count = ctxt->rep_prefix ?
1240 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1241 in_page = (ctxt->eflags & EFLG_DF) ?
1242 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1243 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1244 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1248 rc->pos = rc->end = 0;
1249 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1254 memcpy(dest, rc->data + rc->pos, size);
1259 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1260 u16 index, struct desc_struct *desc)
1265 ctxt->ops->get_idt(ctxt, &dt);
1267 if (dt.size < index * 8 + 7)
1268 return emulate_gp(ctxt, index << 3 | 0x2);
1270 addr = dt.address + index * 8;
1271 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1275 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1276 u16 selector, struct desc_ptr *dt)
1278 struct x86_emulate_ops *ops = ctxt->ops;
1280 if (selector & 1 << 2) {
1281 struct desc_struct desc;
1284 memset (dt, 0, sizeof *dt);
1285 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1288 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1289 dt->address = get_desc_base(&desc);
1291 ops->get_gdt(ctxt, dt);
1294 /* allowed just for 8 bytes segments */
1295 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1296 u16 selector, struct desc_struct *desc,
1300 u16 index = selector >> 3;
1303 get_descriptor_table_ptr(ctxt, selector, &dt);
1305 if (dt.size < index * 8 + 7)
1306 return emulate_gp(ctxt, selector & 0xfffc);
1308 *desc_addr_p = addr = dt.address + index * 8;
1309 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1313 /* allowed just for 8 bytes segments */
1314 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1315 u16 selector, struct desc_struct *desc)
1318 u16 index = selector >> 3;
1321 get_descriptor_table_ptr(ctxt, selector, &dt);
1323 if (dt.size < index * 8 + 7)
1324 return emulate_gp(ctxt, selector & 0xfffc);
1326 addr = dt.address + index * 8;
1327 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1331 /* Does not support long mode */
1332 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1333 u16 selector, int seg)
1335 struct desc_struct seg_desc, old_desc;
1337 unsigned err_vec = GP_VECTOR;
1339 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1343 memset(&seg_desc, 0, sizeof seg_desc);
1345 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1346 || ctxt->mode == X86EMUL_MODE_REAL) {
1347 /* set real mode segment descriptor */
1348 set_desc_base(&seg_desc, selector << 4);
1349 set_desc_limit(&seg_desc, 0xffff);
1353 if (ctxt->mode == X86EMUL_MODE_VM86)
1359 cpl = ctxt->ops->cpl(ctxt);
1361 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1362 if ((seg == VCPU_SREG_CS
1363 || (seg == VCPU_SREG_SS
1364 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1365 || seg == VCPU_SREG_TR)
1369 /* TR should be in GDT only */
1370 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1373 if (null_selector) /* for NULL selector skip all following checks */
1376 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1377 if (ret != X86EMUL_CONTINUE)
1380 err_code = selector & 0xfffc;
1381 err_vec = GP_VECTOR;
1383 /* can't load system descriptor into segment selector */
1384 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1388 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1397 * segment is not a writable data segment or segment
1398 * selector's RPL != CPL or segment selector's RPL != CPL
1400 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1404 if (!(seg_desc.type & 8))
1407 if (seg_desc.type & 4) {
1413 if (rpl > cpl || dpl != cpl)
1416 /* CS(RPL) <- CPL */
1417 selector = (selector & 0xfffc) | cpl;
1420 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1422 old_desc = seg_desc;
1423 seg_desc.type |= 2; /* busy */
1424 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1425 sizeof(seg_desc), &ctxt->exception);
1426 if (ret != X86EMUL_CONTINUE)
1429 case VCPU_SREG_LDTR:
1430 if (seg_desc.s || seg_desc.type != 2)
1433 default: /* DS, ES, FS, or GS */
1435 * segment is not a data or readable code segment or
1436 * ((segment is a data or nonconforming code segment)
1437 * and (both RPL and CPL > DPL))
1439 if ((seg_desc.type & 0xa) == 0x8 ||
1440 (((seg_desc.type & 0xc) != 0xc) &&
1441 (rpl > dpl && cpl > dpl)))
1447 /* mark segment as accessed */
1449 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1450 if (ret != X86EMUL_CONTINUE)
1454 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1455 return X86EMUL_CONTINUE;
1457 emulate_exception(ctxt, err_vec, err_code, true);
1458 return X86EMUL_PROPAGATE_FAULT;
1461 static void write_register_operand(struct operand *op)
1463 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1464 switch (op->bytes) {
1466 *(u8 *)op->addr.reg = (u8)op->val;
1469 *(u16 *)op->addr.reg = (u16)op->val;
1472 *op->addr.reg = (u32)op->val;
1473 break; /* 64b: zero-extend */
1475 *op->addr.reg = op->val;
1480 static int writeback(struct x86_emulate_ctxt *ctxt)
1484 switch (ctxt->dst.type) {
1486 write_register_operand(&ctxt->dst);
1489 if (ctxt->lock_prefix)
1490 rc = segmented_cmpxchg(ctxt,
1492 &ctxt->dst.orig_val,
1496 rc = segmented_write(ctxt,
1500 if (rc != X86EMUL_CONTINUE)
1504 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1507 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1515 return X86EMUL_CONTINUE;
1518 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1520 struct segmented_address addr;
1522 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -bytes);
1523 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1524 addr.seg = VCPU_SREG_SS;
1526 return segmented_write(ctxt, addr, data, bytes);
1529 static int em_push(struct x86_emulate_ctxt *ctxt)
1531 /* Disable writeback. */
1532 ctxt->dst.type = OP_NONE;
1533 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1536 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1537 void *dest, int len)
1540 struct segmented_address addr;
1542 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1543 addr.seg = VCPU_SREG_SS;
1544 rc = segmented_read(ctxt, addr, dest, len);
1545 if (rc != X86EMUL_CONTINUE)
1548 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1552 static int em_pop(struct x86_emulate_ctxt *ctxt)
1554 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1557 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1558 void *dest, int len)
1561 unsigned long val, change_mask;
1562 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1563 int cpl = ctxt->ops->cpl(ctxt);
1565 rc = emulate_pop(ctxt, &val, len);
1566 if (rc != X86EMUL_CONTINUE)
1569 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1570 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1572 switch(ctxt->mode) {
1573 case X86EMUL_MODE_PROT64:
1574 case X86EMUL_MODE_PROT32:
1575 case X86EMUL_MODE_PROT16:
1577 change_mask |= EFLG_IOPL;
1579 change_mask |= EFLG_IF;
1581 case X86EMUL_MODE_VM86:
1583 return emulate_gp(ctxt, 0);
1584 change_mask |= EFLG_IF;
1586 default: /* real mode */
1587 change_mask |= (EFLG_IOPL | EFLG_IF);
1591 *(unsigned long *)dest =
1592 (ctxt->eflags & ~change_mask) | (val & change_mask);
1597 static int em_popf(struct x86_emulate_ctxt *ctxt)
1599 ctxt->dst.type = OP_REG;
1600 ctxt->dst.addr.reg = &ctxt->eflags;
1601 ctxt->dst.bytes = ctxt->op_bytes;
1602 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1605 static int em_enter(struct x86_emulate_ctxt *ctxt)
1608 unsigned frame_size = ctxt->src.val;
1609 unsigned nesting_level = ctxt->src2.val & 31;
1612 return X86EMUL_UNHANDLEABLE;
1614 rc = push(ctxt, &ctxt->regs[VCPU_REGS_RBP], stack_size(ctxt));
1615 if (rc != X86EMUL_CONTINUE)
1617 assign_masked(&ctxt->regs[VCPU_REGS_RBP], ctxt->regs[VCPU_REGS_RSP],
1619 assign_masked(&ctxt->regs[VCPU_REGS_RSP],
1620 ctxt->regs[VCPU_REGS_RSP] - frame_size,
1622 return X86EMUL_CONTINUE;
1625 static int em_leave(struct x86_emulate_ctxt *ctxt)
1627 assign_masked(&ctxt->regs[VCPU_REGS_RSP], ctxt->regs[VCPU_REGS_RBP],
1629 return emulate_pop(ctxt, &ctxt->regs[VCPU_REGS_RBP], ctxt->op_bytes);
1632 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1634 int seg = ctxt->src2.val;
1636 ctxt->src.val = get_segment_selector(ctxt, seg);
1638 return em_push(ctxt);
1641 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1643 int seg = ctxt->src2.val;
1644 unsigned long selector;
1647 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1648 if (rc != X86EMUL_CONTINUE)
1651 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1655 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1657 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1658 int rc = X86EMUL_CONTINUE;
1659 int reg = VCPU_REGS_RAX;
1661 while (reg <= VCPU_REGS_RDI) {
1662 (reg == VCPU_REGS_RSP) ?
1663 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1666 if (rc != X86EMUL_CONTINUE)
1675 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1677 ctxt->src.val = (unsigned long)ctxt->eflags;
1678 return em_push(ctxt);
1681 static int em_popa(struct x86_emulate_ctxt *ctxt)
1683 int rc = X86EMUL_CONTINUE;
1684 int reg = VCPU_REGS_RDI;
1686 while (reg >= VCPU_REGS_RAX) {
1687 if (reg == VCPU_REGS_RSP) {
1688 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1693 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1694 if (rc != X86EMUL_CONTINUE)
1701 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1703 struct x86_emulate_ops *ops = ctxt->ops;
1710 /* TODO: Add limit checks */
1711 ctxt->src.val = ctxt->eflags;
1713 if (rc != X86EMUL_CONTINUE)
1716 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1718 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1720 if (rc != X86EMUL_CONTINUE)
1723 ctxt->src.val = ctxt->_eip;
1725 if (rc != X86EMUL_CONTINUE)
1728 ops->get_idt(ctxt, &dt);
1730 eip_addr = dt.address + (irq << 2);
1731 cs_addr = dt.address + (irq << 2) + 2;
1733 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1734 if (rc != X86EMUL_CONTINUE)
1737 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1738 if (rc != X86EMUL_CONTINUE)
1741 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1742 if (rc != X86EMUL_CONTINUE)
1750 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1752 switch(ctxt->mode) {
1753 case X86EMUL_MODE_REAL:
1754 return emulate_int_real(ctxt, irq);
1755 case X86EMUL_MODE_VM86:
1756 case X86EMUL_MODE_PROT16:
1757 case X86EMUL_MODE_PROT32:
1758 case X86EMUL_MODE_PROT64:
1760 /* Protected mode interrupts unimplemented yet */
1761 return X86EMUL_UNHANDLEABLE;
1765 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1767 int rc = X86EMUL_CONTINUE;
1768 unsigned long temp_eip = 0;
1769 unsigned long temp_eflags = 0;
1770 unsigned long cs = 0;
1771 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1772 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1773 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1774 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1776 /* TODO: Add stack limit check */
1778 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1780 if (rc != X86EMUL_CONTINUE)
1783 if (temp_eip & ~0xffff)
1784 return emulate_gp(ctxt, 0);
1786 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1788 if (rc != X86EMUL_CONTINUE)
1791 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1793 if (rc != X86EMUL_CONTINUE)
1796 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1798 if (rc != X86EMUL_CONTINUE)
1801 ctxt->_eip = temp_eip;
1804 if (ctxt->op_bytes == 4)
1805 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1806 else if (ctxt->op_bytes == 2) {
1807 ctxt->eflags &= ~0xffff;
1808 ctxt->eflags |= temp_eflags;
1811 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1812 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1817 static int em_iret(struct x86_emulate_ctxt *ctxt)
1819 switch(ctxt->mode) {
1820 case X86EMUL_MODE_REAL:
1821 return emulate_iret_real(ctxt);
1822 case X86EMUL_MODE_VM86:
1823 case X86EMUL_MODE_PROT16:
1824 case X86EMUL_MODE_PROT32:
1825 case X86EMUL_MODE_PROT64:
1827 /* iret from protected mode unimplemented yet */
1828 return X86EMUL_UNHANDLEABLE;
1832 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1837 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1839 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1840 if (rc != X86EMUL_CONTINUE)
1844 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1845 return X86EMUL_CONTINUE;
1848 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1850 switch (ctxt->modrm_reg) {
1852 emulate_2op_SrcB(ctxt, "rol");
1855 emulate_2op_SrcB(ctxt, "ror");
1858 emulate_2op_SrcB(ctxt, "rcl");
1861 emulate_2op_SrcB(ctxt, "rcr");
1863 case 4: /* sal/shl */
1864 case 6: /* sal/shl */
1865 emulate_2op_SrcB(ctxt, "sal");
1868 emulate_2op_SrcB(ctxt, "shr");
1871 emulate_2op_SrcB(ctxt, "sar");
1874 return X86EMUL_CONTINUE;
1877 static int em_not(struct x86_emulate_ctxt *ctxt)
1879 ctxt->dst.val = ~ctxt->dst.val;
1880 return X86EMUL_CONTINUE;
1883 static int em_neg(struct x86_emulate_ctxt *ctxt)
1885 emulate_1op(ctxt, "neg");
1886 return X86EMUL_CONTINUE;
1889 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1893 emulate_1op_rax_rdx(ctxt, "mul", ex);
1894 return X86EMUL_CONTINUE;
1897 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1901 emulate_1op_rax_rdx(ctxt, "imul", ex);
1902 return X86EMUL_CONTINUE;
1905 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1909 emulate_1op_rax_rdx(ctxt, "div", de);
1911 return emulate_de(ctxt);
1912 return X86EMUL_CONTINUE;
1915 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1919 emulate_1op_rax_rdx(ctxt, "idiv", de);
1921 return emulate_de(ctxt);
1922 return X86EMUL_CONTINUE;
1925 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1927 int rc = X86EMUL_CONTINUE;
1929 switch (ctxt->modrm_reg) {
1931 emulate_1op(ctxt, "inc");
1934 emulate_1op(ctxt, "dec");
1936 case 2: /* call near abs */ {
1938 old_eip = ctxt->_eip;
1939 ctxt->_eip = ctxt->src.val;
1940 ctxt->src.val = old_eip;
1944 case 4: /* jmp abs */
1945 ctxt->_eip = ctxt->src.val;
1947 case 5: /* jmp far */
1948 rc = em_jmp_far(ctxt);
1957 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1959 u64 old = ctxt->dst.orig_val64;
1961 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1962 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1963 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1964 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1965 ctxt->eflags &= ~EFLG_ZF;
1967 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1968 (u32) ctxt->regs[VCPU_REGS_RBX];
1970 ctxt->eflags |= EFLG_ZF;
1972 return X86EMUL_CONTINUE;
1975 static int em_ret(struct x86_emulate_ctxt *ctxt)
1977 ctxt->dst.type = OP_REG;
1978 ctxt->dst.addr.reg = &ctxt->_eip;
1979 ctxt->dst.bytes = ctxt->op_bytes;
1980 return em_pop(ctxt);
1983 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1988 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1989 if (rc != X86EMUL_CONTINUE)
1991 if (ctxt->op_bytes == 4)
1992 ctxt->_eip = (u32)ctxt->_eip;
1993 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1994 if (rc != X86EMUL_CONTINUE)
1996 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2000 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2002 /* Save real source value, then compare EAX against destination. */
2003 ctxt->src.orig_val = ctxt->src.val;
2004 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
2005 emulate_2op_SrcV(ctxt, "cmp");
2007 if (ctxt->eflags & EFLG_ZF) {
2008 /* Success: write back to memory. */
2009 ctxt->dst.val = ctxt->src.orig_val;
2011 /* Failure: write the value we saw to EAX. */
2012 ctxt->dst.type = OP_REG;
2013 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
2015 return X86EMUL_CONTINUE;
2018 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2020 int seg = ctxt->src2.val;
2024 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2026 rc = load_segment_descriptor(ctxt, sel, seg);
2027 if (rc != X86EMUL_CONTINUE)
2030 ctxt->dst.val = ctxt->src.val;
2035 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2036 struct desc_struct *cs, struct desc_struct *ss)
2038 cs->l = 0; /* will be adjusted later */
2039 set_desc_base(cs, 0); /* flat segment */
2040 cs->g = 1; /* 4kb granularity */
2041 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2042 cs->type = 0x0b; /* Read, Execute, Accessed */
2044 cs->dpl = 0; /* will be adjusted later */
2049 set_desc_base(ss, 0); /* flat segment */
2050 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2051 ss->g = 1; /* 4kb granularity */
2053 ss->type = 0x03; /* Read/Write, Accessed */
2054 ss->d = 1; /* 32bit stack segment */
2061 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2063 u32 eax, ebx, ecx, edx;
2066 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2067 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2068 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2069 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2072 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2074 struct x86_emulate_ops *ops = ctxt->ops;
2075 u32 eax, ebx, ecx, edx;
2078 * syscall should always be enabled in longmode - so only become
2079 * vendor specific (cpuid) if other modes are active...
2081 if (ctxt->mode == X86EMUL_MODE_PROT64)
2086 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2088 * Intel ("GenuineIntel")
2089 * remark: Intel CPUs only support "syscall" in 64bit
2090 * longmode. Also an 64bit guest with a
2091 * 32bit compat-app running will #UD !! While this
2092 * behaviour can be fixed (by emulating) into AMD
2093 * response - CPUs of AMD can't behave like Intel.
2095 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2096 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2097 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2100 /* AMD ("AuthenticAMD") */
2101 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2102 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2103 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2106 /* AMD ("AMDisbetter!") */
2107 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2108 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2109 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2112 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2116 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2118 struct x86_emulate_ops *ops = ctxt->ops;
2119 struct desc_struct cs, ss;
2124 /* syscall is not available in real mode */
2125 if (ctxt->mode == X86EMUL_MODE_REAL ||
2126 ctxt->mode == X86EMUL_MODE_VM86)
2127 return emulate_ud(ctxt);
2129 if (!(em_syscall_is_enabled(ctxt)))
2130 return emulate_ud(ctxt);
2132 ops->get_msr(ctxt, MSR_EFER, &efer);
2133 setup_syscalls_segments(ctxt, &cs, &ss);
2135 if (!(efer & EFER_SCE))
2136 return emulate_ud(ctxt);
2138 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2140 cs_sel = (u16)(msr_data & 0xfffc);
2141 ss_sel = (u16)(msr_data + 8);
2143 if (efer & EFER_LMA) {
2147 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2148 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2150 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
2151 if (efer & EFER_LMA) {
2152 #ifdef CONFIG_X86_64
2153 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2156 ctxt->mode == X86EMUL_MODE_PROT64 ?
2157 MSR_LSTAR : MSR_CSTAR, &msr_data);
2158 ctxt->_eip = msr_data;
2160 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2161 ctxt->eflags &= ~(msr_data | EFLG_RF);
2165 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2166 ctxt->_eip = (u32)msr_data;
2168 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2171 return X86EMUL_CONTINUE;
2174 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2176 struct x86_emulate_ops *ops = ctxt->ops;
2177 struct desc_struct cs, ss;
2182 ops->get_msr(ctxt, MSR_EFER, &efer);
2183 /* inject #GP if in real mode */
2184 if (ctxt->mode == X86EMUL_MODE_REAL)
2185 return emulate_gp(ctxt, 0);
2188 * Not recognized on AMD in compat mode (but is recognized in legacy
2191 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2192 && !vendor_intel(ctxt))
2193 return emulate_ud(ctxt);
2195 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2196 * Therefore, we inject an #UD.
2198 if (ctxt->mode == X86EMUL_MODE_PROT64)
2199 return emulate_ud(ctxt);
2201 setup_syscalls_segments(ctxt, &cs, &ss);
2203 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2204 switch (ctxt->mode) {
2205 case X86EMUL_MODE_PROT32:
2206 if ((msr_data & 0xfffc) == 0x0)
2207 return emulate_gp(ctxt, 0);
2209 case X86EMUL_MODE_PROT64:
2210 if (msr_data == 0x0)
2211 return emulate_gp(ctxt, 0);
2215 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2216 cs_sel = (u16)msr_data;
2217 cs_sel &= ~SELECTOR_RPL_MASK;
2218 ss_sel = cs_sel + 8;
2219 ss_sel &= ~SELECTOR_RPL_MASK;
2220 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2225 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2226 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2228 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2229 ctxt->_eip = msr_data;
2231 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2232 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2234 return X86EMUL_CONTINUE;
2237 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2239 struct x86_emulate_ops *ops = ctxt->ops;
2240 struct desc_struct cs, ss;
2243 u16 cs_sel = 0, ss_sel = 0;
2245 /* inject #GP if in real mode or Virtual 8086 mode */
2246 if (ctxt->mode == X86EMUL_MODE_REAL ||
2247 ctxt->mode == X86EMUL_MODE_VM86)
2248 return emulate_gp(ctxt, 0);
2250 setup_syscalls_segments(ctxt, &cs, &ss);
2252 if ((ctxt->rex_prefix & 0x8) != 0x0)
2253 usermode = X86EMUL_MODE_PROT64;
2255 usermode = X86EMUL_MODE_PROT32;
2259 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2261 case X86EMUL_MODE_PROT32:
2262 cs_sel = (u16)(msr_data + 16);
2263 if ((msr_data & 0xfffc) == 0x0)
2264 return emulate_gp(ctxt, 0);
2265 ss_sel = (u16)(msr_data + 24);
2267 case X86EMUL_MODE_PROT64:
2268 cs_sel = (u16)(msr_data + 32);
2269 if (msr_data == 0x0)
2270 return emulate_gp(ctxt, 0);
2271 ss_sel = cs_sel + 8;
2276 cs_sel |= SELECTOR_RPL_MASK;
2277 ss_sel |= SELECTOR_RPL_MASK;
2279 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2280 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2282 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2283 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2285 return X86EMUL_CONTINUE;
2288 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2291 if (ctxt->mode == X86EMUL_MODE_REAL)
2293 if (ctxt->mode == X86EMUL_MODE_VM86)
2295 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2296 return ctxt->ops->cpl(ctxt) > iopl;
2299 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2302 struct x86_emulate_ops *ops = ctxt->ops;
2303 struct desc_struct tr_seg;
2306 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2307 unsigned mask = (1 << len) - 1;
2310 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2313 if (desc_limit_scaled(&tr_seg) < 103)
2315 base = get_desc_base(&tr_seg);
2316 #ifdef CONFIG_X86_64
2317 base |= ((u64)base3) << 32;
2319 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2320 if (r != X86EMUL_CONTINUE)
2322 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2324 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2325 if (r != X86EMUL_CONTINUE)
2327 if ((perm >> bit_idx) & mask)
2332 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2338 if (emulator_bad_iopl(ctxt))
2339 if (!emulator_io_port_access_allowed(ctxt, port, len))
2342 ctxt->perm_ok = true;
2347 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2348 struct tss_segment_16 *tss)
2350 tss->ip = ctxt->_eip;
2351 tss->flag = ctxt->eflags;
2352 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2353 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2354 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2355 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2356 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2357 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2358 tss->si = ctxt->regs[VCPU_REGS_RSI];
2359 tss->di = ctxt->regs[VCPU_REGS_RDI];
2361 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2362 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2363 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2364 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2365 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2368 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2369 struct tss_segment_16 *tss)
2373 ctxt->_eip = tss->ip;
2374 ctxt->eflags = tss->flag | 2;
2375 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2376 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2377 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2378 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2379 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2380 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2381 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2382 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2385 * SDM says that segment selectors are loaded before segment
2388 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2389 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2390 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2391 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2392 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2395 * Now load segment descriptors. If fault happens at this stage
2396 * it is handled in a context of new task
2398 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2399 if (ret != X86EMUL_CONTINUE)
2401 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2402 if (ret != X86EMUL_CONTINUE)
2404 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2405 if (ret != X86EMUL_CONTINUE)
2407 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2408 if (ret != X86EMUL_CONTINUE)
2410 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2411 if (ret != X86EMUL_CONTINUE)
2414 return X86EMUL_CONTINUE;
2417 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2418 u16 tss_selector, u16 old_tss_sel,
2419 ulong old_tss_base, struct desc_struct *new_desc)
2421 struct x86_emulate_ops *ops = ctxt->ops;
2422 struct tss_segment_16 tss_seg;
2424 u32 new_tss_base = get_desc_base(new_desc);
2426 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2428 if (ret != X86EMUL_CONTINUE)
2429 /* FIXME: need to provide precise fault address */
2432 save_state_to_tss16(ctxt, &tss_seg);
2434 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2436 if (ret != X86EMUL_CONTINUE)
2437 /* FIXME: need to provide precise fault address */
2440 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2442 if (ret != X86EMUL_CONTINUE)
2443 /* FIXME: need to provide precise fault address */
2446 if (old_tss_sel != 0xffff) {
2447 tss_seg.prev_task_link = old_tss_sel;
2449 ret = ops->write_std(ctxt, new_tss_base,
2450 &tss_seg.prev_task_link,
2451 sizeof tss_seg.prev_task_link,
2453 if (ret != X86EMUL_CONTINUE)
2454 /* FIXME: need to provide precise fault address */
2458 return load_state_from_tss16(ctxt, &tss_seg);
2461 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2462 struct tss_segment_32 *tss)
2464 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2465 tss->eip = ctxt->_eip;
2466 tss->eflags = ctxt->eflags;
2467 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2468 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2469 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2470 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2471 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2472 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2473 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2474 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2476 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2477 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2478 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2479 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2480 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2481 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2482 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2485 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2486 struct tss_segment_32 *tss)
2490 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2491 return emulate_gp(ctxt, 0);
2492 ctxt->_eip = tss->eip;
2493 ctxt->eflags = tss->eflags | 2;
2495 /* General purpose registers */
2496 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2497 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2498 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2499 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2500 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2501 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2502 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2503 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2506 * SDM says that segment selectors are loaded before segment
2509 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2510 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2511 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2512 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2513 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2514 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2515 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2518 * If we're switching between Protected Mode and VM86, we need to make
2519 * sure to update the mode before loading the segment descriptors so
2520 * that the selectors are interpreted correctly.
2522 * Need to get rflags to the vcpu struct immediately because it
2523 * influences the CPL which is checked at least when loading the segment
2524 * descriptors and when pushing an error code to the new kernel stack.
2526 * TODO Introduce a separate ctxt->ops->set_cpl callback
2528 if (ctxt->eflags & X86_EFLAGS_VM)
2529 ctxt->mode = X86EMUL_MODE_VM86;
2531 ctxt->mode = X86EMUL_MODE_PROT32;
2533 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2536 * Now load segment descriptors. If fault happenes at this stage
2537 * it is handled in a context of new task
2539 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2540 if (ret != X86EMUL_CONTINUE)
2542 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2543 if (ret != X86EMUL_CONTINUE)
2545 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2546 if (ret != X86EMUL_CONTINUE)
2548 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2549 if (ret != X86EMUL_CONTINUE)
2551 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2552 if (ret != X86EMUL_CONTINUE)
2554 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2555 if (ret != X86EMUL_CONTINUE)
2557 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2558 if (ret != X86EMUL_CONTINUE)
2561 return X86EMUL_CONTINUE;
2564 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2565 u16 tss_selector, u16 old_tss_sel,
2566 ulong old_tss_base, struct desc_struct *new_desc)
2568 struct x86_emulate_ops *ops = ctxt->ops;
2569 struct tss_segment_32 tss_seg;
2571 u32 new_tss_base = get_desc_base(new_desc);
2573 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2575 if (ret != X86EMUL_CONTINUE)
2576 /* FIXME: need to provide precise fault address */
2579 save_state_to_tss32(ctxt, &tss_seg);
2581 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2583 if (ret != X86EMUL_CONTINUE)
2584 /* FIXME: need to provide precise fault address */
2587 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2589 if (ret != X86EMUL_CONTINUE)
2590 /* FIXME: need to provide precise fault address */
2593 if (old_tss_sel != 0xffff) {
2594 tss_seg.prev_task_link = old_tss_sel;
2596 ret = ops->write_std(ctxt, new_tss_base,
2597 &tss_seg.prev_task_link,
2598 sizeof tss_seg.prev_task_link,
2600 if (ret != X86EMUL_CONTINUE)
2601 /* FIXME: need to provide precise fault address */
2605 return load_state_from_tss32(ctxt, &tss_seg);
2608 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2609 u16 tss_selector, int idt_index, int reason,
2610 bool has_error_code, u32 error_code)
2612 struct x86_emulate_ops *ops = ctxt->ops;
2613 struct desc_struct curr_tss_desc, next_tss_desc;
2615 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2616 ulong old_tss_base =
2617 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2621 /* FIXME: old_tss_base == ~0 ? */
2623 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2624 if (ret != X86EMUL_CONTINUE)
2626 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2627 if (ret != X86EMUL_CONTINUE)
2630 /* FIXME: check that next_tss_desc is tss */
2633 * Check privileges. The three cases are task switch caused by...
2635 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2636 * 2. Exception/IRQ/iret: No check is performed
2637 * 3. jmp/call to TSS: Check against DPL of the TSS
2639 if (reason == TASK_SWITCH_GATE) {
2640 if (idt_index != -1) {
2641 /* Software interrupts */
2642 struct desc_struct task_gate_desc;
2645 ret = read_interrupt_descriptor(ctxt, idt_index,
2647 if (ret != X86EMUL_CONTINUE)
2650 dpl = task_gate_desc.dpl;
2651 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2652 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2654 } else if (reason != TASK_SWITCH_IRET) {
2655 int dpl = next_tss_desc.dpl;
2656 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2657 return emulate_gp(ctxt, tss_selector);
2661 desc_limit = desc_limit_scaled(&next_tss_desc);
2662 if (!next_tss_desc.p ||
2663 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2664 desc_limit < 0x2b)) {
2665 emulate_ts(ctxt, tss_selector & 0xfffc);
2666 return X86EMUL_PROPAGATE_FAULT;
2669 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2670 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2671 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2674 if (reason == TASK_SWITCH_IRET)
2675 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2677 /* set back link to prev task only if NT bit is set in eflags
2678 note that old_tss_sel is not used after this point */
2679 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2680 old_tss_sel = 0xffff;
2682 if (next_tss_desc.type & 8)
2683 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2684 old_tss_base, &next_tss_desc);
2686 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2687 old_tss_base, &next_tss_desc);
2688 if (ret != X86EMUL_CONTINUE)
2691 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2692 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2694 if (reason != TASK_SWITCH_IRET) {
2695 next_tss_desc.type |= (1 << 1); /* set busy flag */
2696 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2699 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2700 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2702 if (has_error_code) {
2703 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2704 ctxt->lock_prefix = 0;
2705 ctxt->src.val = (unsigned long) error_code;
2706 ret = em_push(ctxt);
2712 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2713 u16 tss_selector, int idt_index, int reason,
2714 bool has_error_code, u32 error_code)
2718 ctxt->_eip = ctxt->eip;
2719 ctxt->dst.type = OP_NONE;
2721 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2722 has_error_code, error_code);
2724 if (rc == X86EMUL_CONTINUE)
2725 ctxt->eip = ctxt->_eip;
2727 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2730 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2731 int reg, struct operand *op)
2733 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2735 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2736 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2737 op->addr.mem.seg = seg;
2740 static int em_das(struct x86_emulate_ctxt *ctxt)
2743 bool af, cf, old_cf;
2745 cf = ctxt->eflags & X86_EFLAGS_CF;
2751 af = ctxt->eflags & X86_EFLAGS_AF;
2752 if ((al & 0x0f) > 9 || af) {
2754 cf = old_cf | (al >= 250);
2759 if (old_al > 0x99 || old_cf) {
2765 /* Set PF, ZF, SF */
2766 ctxt->src.type = OP_IMM;
2768 ctxt->src.bytes = 1;
2769 emulate_2op_SrcV(ctxt, "or");
2770 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2772 ctxt->eflags |= X86_EFLAGS_CF;
2774 ctxt->eflags |= X86_EFLAGS_AF;
2775 return X86EMUL_CONTINUE;
2778 static int em_call(struct x86_emulate_ctxt *ctxt)
2780 long rel = ctxt->src.val;
2782 ctxt->src.val = (unsigned long)ctxt->_eip;
2784 return em_push(ctxt);
2787 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2793 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2794 old_eip = ctxt->_eip;
2796 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2797 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2798 return X86EMUL_CONTINUE;
2801 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2803 ctxt->src.val = old_cs;
2805 if (rc != X86EMUL_CONTINUE)
2808 ctxt->src.val = old_eip;
2809 return em_push(ctxt);
2812 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2816 ctxt->dst.type = OP_REG;
2817 ctxt->dst.addr.reg = &ctxt->_eip;
2818 ctxt->dst.bytes = ctxt->op_bytes;
2819 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2820 if (rc != X86EMUL_CONTINUE)
2822 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2823 return X86EMUL_CONTINUE;
2826 static int em_add(struct x86_emulate_ctxt *ctxt)
2828 emulate_2op_SrcV(ctxt, "add");
2829 return X86EMUL_CONTINUE;
2832 static int em_or(struct x86_emulate_ctxt *ctxt)
2834 emulate_2op_SrcV(ctxt, "or");
2835 return X86EMUL_CONTINUE;
2838 static int em_adc(struct x86_emulate_ctxt *ctxt)
2840 emulate_2op_SrcV(ctxt, "adc");
2841 return X86EMUL_CONTINUE;
2844 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2846 emulate_2op_SrcV(ctxt, "sbb");
2847 return X86EMUL_CONTINUE;
2850 static int em_and(struct x86_emulate_ctxt *ctxt)
2852 emulate_2op_SrcV(ctxt, "and");
2853 return X86EMUL_CONTINUE;
2856 static int em_sub(struct x86_emulate_ctxt *ctxt)
2858 emulate_2op_SrcV(ctxt, "sub");
2859 return X86EMUL_CONTINUE;
2862 static int em_xor(struct x86_emulate_ctxt *ctxt)
2864 emulate_2op_SrcV(ctxt, "xor");
2865 return X86EMUL_CONTINUE;
2868 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2870 emulate_2op_SrcV(ctxt, "cmp");
2871 /* Disable writeback. */
2872 ctxt->dst.type = OP_NONE;
2873 return X86EMUL_CONTINUE;
2876 static int em_test(struct x86_emulate_ctxt *ctxt)
2878 emulate_2op_SrcV(ctxt, "test");
2879 /* Disable writeback. */
2880 ctxt->dst.type = OP_NONE;
2881 return X86EMUL_CONTINUE;
2884 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2886 /* Write back the register source. */
2887 ctxt->src.val = ctxt->dst.val;
2888 write_register_operand(&ctxt->src);
2890 /* Write back the memory destination with implicit LOCK prefix. */
2891 ctxt->dst.val = ctxt->src.orig_val;
2892 ctxt->lock_prefix = 1;
2893 return X86EMUL_CONTINUE;
2896 static int em_imul(struct x86_emulate_ctxt *ctxt)
2898 emulate_2op_SrcV_nobyte(ctxt, "imul");
2899 return X86EMUL_CONTINUE;
2902 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2904 ctxt->dst.val = ctxt->src2.val;
2905 return em_imul(ctxt);
2908 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2910 ctxt->dst.type = OP_REG;
2911 ctxt->dst.bytes = ctxt->src.bytes;
2912 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2913 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2915 return X86EMUL_CONTINUE;
2918 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2922 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2923 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2924 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2925 return X86EMUL_CONTINUE;
2928 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2932 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
2933 return emulate_gp(ctxt, 0);
2934 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
2935 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
2936 return X86EMUL_CONTINUE;
2939 static int em_mov(struct x86_emulate_ctxt *ctxt)
2941 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2942 return X86EMUL_CONTINUE;
2945 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2947 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2948 return emulate_gp(ctxt, 0);
2950 /* Disable writeback. */
2951 ctxt->dst.type = OP_NONE;
2952 return X86EMUL_CONTINUE;
2955 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2959 if (ctxt->mode == X86EMUL_MODE_PROT64)
2960 val = ctxt->src.val & ~0ULL;
2962 val = ctxt->src.val & ~0U;
2964 /* #UD condition is already handled. */
2965 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2966 return emulate_gp(ctxt, 0);
2968 /* Disable writeback. */
2969 ctxt->dst.type = OP_NONE;
2970 return X86EMUL_CONTINUE;
2973 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2977 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
2978 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
2979 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
2980 return emulate_gp(ctxt, 0);
2982 return X86EMUL_CONTINUE;
2985 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2989 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
2990 return emulate_gp(ctxt, 0);
2992 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
2993 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
2994 return X86EMUL_CONTINUE;
2997 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2999 if (ctxt->modrm_reg > VCPU_SREG_GS)
3000 return emulate_ud(ctxt);
3002 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3003 return X86EMUL_CONTINUE;
3006 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3008 u16 sel = ctxt->src.val;
3010 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3011 return emulate_ud(ctxt);
3013 if (ctxt->modrm_reg == VCPU_SREG_SS)
3014 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3016 /* Disable writeback. */
3017 ctxt->dst.type = OP_NONE;
3018 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3021 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3023 u16 sel = ctxt->src.val;
3025 /* Disable writeback. */
3026 ctxt->dst.type = OP_NONE;
3027 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3030 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3032 u16 sel = ctxt->src.val;
3034 /* Disable writeback. */
3035 ctxt->dst.type = OP_NONE;
3036 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3039 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3044 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3045 if (rc == X86EMUL_CONTINUE)
3046 ctxt->ops->invlpg(ctxt, linear);
3047 /* Disable writeback. */
3048 ctxt->dst.type = OP_NONE;
3049 return X86EMUL_CONTINUE;
3052 static int em_clts(struct x86_emulate_ctxt *ctxt)
3056 cr0 = ctxt->ops->get_cr(ctxt, 0);
3058 ctxt->ops->set_cr(ctxt, 0, cr0);
3059 return X86EMUL_CONTINUE;
3062 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3066 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3067 return X86EMUL_UNHANDLEABLE;
3069 rc = ctxt->ops->fix_hypercall(ctxt);
3070 if (rc != X86EMUL_CONTINUE)
3073 /* Let the processor re-execute the fixed hypercall */
3074 ctxt->_eip = ctxt->eip;
3075 /* Disable writeback. */
3076 ctxt->dst.type = OP_NONE;
3077 return X86EMUL_CONTINUE;
3080 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3081 void (*get)(struct x86_emulate_ctxt *ctxt,
3082 struct desc_ptr *ptr))
3084 struct desc_ptr desc_ptr;
3086 if (ctxt->mode == X86EMUL_MODE_PROT64)
3088 get(ctxt, &desc_ptr);
3089 if (ctxt->op_bytes == 2) {
3091 desc_ptr.address &= 0x00ffffff;
3093 /* Disable writeback. */
3094 ctxt->dst.type = OP_NONE;
3095 return segmented_write(ctxt, ctxt->dst.addr.mem,
3096 &desc_ptr, 2 + ctxt->op_bytes);
3099 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3101 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3104 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3106 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3109 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3111 struct desc_ptr desc_ptr;
3114 if (ctxt->mode == X86EMUL_MODE_PROT64)
3116 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3117 &desc_ptr.size, &desc_ptr.address,
3119 if (rc != X86EMUL_CONTINUE)
3121 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3122 /* Disable writeback. */
3123 ctxt->dst.type = OP_NONE;
3124 return X86EMUL_CONTINUE;
3127 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3131 rc = ctxt->ops->fix_hypercall(ctxt);
3133 /* Disable writeback. */
3134 ctxt->dst.type = OP_NONE;
3138 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3140 struct desc_ptr desc_ptr;
3143 if (ctxt->mode == X86EMUL_MODE_PROT64)
3145 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3146 &desc_ptr.size, &desc_ptr.address,
3148 if (rc != X86EMUL_CONTINUE)
3150 ctxt->ops->set_idt(ctxt, &desc_ptr);
3151 /* Disable writeback. */
3152 ctxt->dst.type = OP_NONE;
3153 return X86EMUL_CONTINUE;
3156 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3158 ctxt->dst.bytes = 2;
3159 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3160 return X86EMUL_CONTINUE;
3163 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3165 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3166 | (ctxt->src.val & 0x0f));
3167 ctxt->dst.type = OP_NONE;
3168 return X86EMUL_CONTINUE;
3171 static int em_loop(struct x86_emulate_ctxt *ctxt)
3173 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3174 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
3175 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3176 jmp_rel(ctxt, ctxt->src.val);
3178 return X86EMUL_CONTINUE;
3181 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3183 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
3184 jmp_rel(ctxt, ctxt->src.val);
3186 return X86EMUL_CONTINUE;
3189 static int em_in(struct x86_emulate_ctxt *ctxt)
3191 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3193 return X86EMUL_IO_NEEDED;
3195 return X86EMUL_CONTINUE;
3198 static int em_out(struct x86_emulate_ctxt *ctxt)
3200 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3202 /* Disable writeback. */
3203 ctxt->dst.type = OP_NONE;
3204 return X86EMUL_CONTINUE;
3207 static int em_cli(struct x86_emulate_ctxt *ctxt)
3209 if (emulator_bad_iopl(ctxt))
3210 return emulate_gp(ctxt, 0);
3212 ctxt->eflags &= ~X86_EFLAGS_IF;
3213 return X86EMUL_CONTINUE;
3216 static int em_sti(struct x86_emulate_ctxt *ctxt)
3218 if (emulator_bad_iopl(ctxt))
3219 return emulate_gp(ctxt, 0);
3221 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3222 ctxt->eflags |= X86_EFLAGS_IF;
3223 return X86EMUL_CONTINUE;
3226 static int em_bt(struct x86_emulate_ctxt *ctxt)
3228 /* Disable writeback. */
3229 ctxt->dst.type = OP_NONE;
3230 /* only subword offset */
3231 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3233 emulate_2op_SrcV_nobyte(ctxt, "bt");
3234 return X86EMUL_CONTINUE;
3237 static int em_bts(struct x86_emulate_ctxt *ctxt)
3239 emulate_2op_SrcV_nobyte(ctxt, "bts");
3240 return X86EMUL_CONTINUE;
3243 static int em_btr(struct x86_emulate_ctxt *ctxt)
3245 emulate_2op_SrcV_nobyte(ctxt, "btr");
3246 return X86EMUL_CONTINUE;
3249 static int em_btc(struct x86_emulate_ctxt *ctxt)
3251 emulate_2op_SrcV_nobyte(ctxt, "btc");
3252 return X86EMUL_CONTINUE;
3255 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3257 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3258 return X86EMUL_CONTINUE;
3261 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3263 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3264 return X86EMUL_CONTINUE;
3267 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3269 u32 eax, ebx, ecx, edx;
3271 eax = ctxt->regs[VCPU_REGS_RAX];
3272 ecx = ctxt->regs[VCPU_REGS_RCX];
3273 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3274 ctxt->regs[VCPU_REGS_RAX] = eax;
3275 ctxt->regs[VCPU_REGS_RBX] = ebx;
3276 ctxt->regs[VCPU_REGS_RCX] = ecx;
3277 ctxt->regs[VCPU_REGS_RDX] = edx;
3278 return X86EMUL_CONTINUE;
3281 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3283 ctxt->regs[VCPU_REGS_RAX] &= ~0xff00UL;
3284 ctxt->regs[VCPU_REGS_RAX] |= (ctxt->eflags & 0xff) << 8;
3285 return X86EMUL_CONTINUE;
3288 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3290 switch (ctxt->op_bytes) {
3291 #ifdef CONFIG_X86_64
3293 asm("bswap %0" : "+r"(ctxt->dst.val));
3297 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3300 return X86EMUL_CONTINUE;
3303 static bool valid_cr(int nr)
3315 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3317 if (!valid_cr(ctxt->modrm_reg))
3318 return emulate_ud(ctxt);
3320 return X86EMUL_CONTINUE;
3323 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3325 u64 new_val = ctxt->src.val64;
3326 int cr = ctxt->modrm_reg;
3329 static u64 cr_reserved_bits[] = {
3330 0xffffffff00000000ULL,
3331 0, 0, 0, /* CR3 checked later */
3338 return emulate_ud(ctxt);
3340 if (new_val & cr_reserved_bits[cr])
3341 return emulate_gp(ctxt, 0);
3346 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3347 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3348 return emulate_gp(ctxt, 0);
3350 cr4 = ctxt->ops->get_cr(ctxt, 4);
3351 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3353 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3354 !(cr4 & X86_CR4_PAE))
3355 return emulate_gp(ctxt, 0);
3362 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3363 if (efer & EFER_LMA)
3364 rsvd = CR3_L_MODE_RESERVED_BITS;
3365 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3366 rsvd = CR3_PAE_RESERVED_BITS;
3367 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3368 rsvd = CR3_NONPAE_RESERVED_BITS;
3371 return emulate_gp(ctxt, 0);
3376 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3378 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3379 return emulate_gp(ctxt, 0);
3385 return X86EMUL_CONTINUE;
3388 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3392 ctxt->ops->get_dr(ctxt, 7, &dr7);
3394 /* Check if DR7.Global_Enable is set */
3395 return dr7 & (1 << 13);
3398 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3400 int dr = ctxt->modrm_reg;
3404 return emulate_ud(ctxt);
3406 cr4 = ctxt->ops->get_cr(ctxt, 4);
3407 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3408 return emulate_ud(ctxt);
3410 if (check_dr7_gd(ctxt))
3411 return emulate_db(ctxt);
3413 return X86EMUL_CONTINUE;
3416 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3418 u64 new_val = ctxt->src.val64;
3419 int dr = ctxt->modrm_reg;
3421 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3422 return emulate_gp(ctxt, 0);
3424 return check_dr_read(ctxt);
3427 static int check_svme(struct x86_emulate_ctxt *ctxt)
3431 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3433 if (!(efer & EFER_SVME))
3434 return emulate_ud(ctxt);
3436 return X86EMUL_CONTINUE;
3439 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3441 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3443 /* Valid physical address? */
3444 if (rax & 0xffff000000000000ULL)
3445 return emulate_gp(ctxt, 0);
3447 return check_svme(ctxt);
3450 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3452 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3454 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3455 return emulate_ud(ctxt);
3457 return X86EMUL_CONTINUE;
3460 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3462 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3463 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3465 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3467 return emulate_gp(ctxt, 0);
3469 return X86EMUL_CONTINUE;
3472 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3474 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3475 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3476 return emulate_gp(ctxt, 0);
3478 return X86EMUL_CONTINUE;
3481 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3483 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3484 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3485 return emulate_gp(ctxt, 0);
3487 return X86EMUL_CONTINUE;
3490 #define D(_y) { .flags = (_y) }
3491 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3492 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3493 .check_perm = (_p) }
3495 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3496 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3497 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3498 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3499 #define II(_f, _e, _i) \
3500 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3501 #define IIP(_f, _e, _i, _p) \
3502 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3503 .check_perm = (_p) }
3504 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3506 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3507 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3508 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3509 #define I2bvIP(_f, _e, _i, _p) \
3510 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3512 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3513 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3514 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3516 static struct opcode group7_rm1[] = {
3517 DI(SrcNone | Priv, monitor),
3518 DI(SrcNone | Priv, mwait),
3522 static struct opcode group7_rm3[] = {
3523 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3524 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3525 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3526 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3527 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3528 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3529 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3530 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3533 static struct opcode group7_rm7[] = {
3535 DIP(SrcNone, rdtscp, check_rdtsc),
3539 static struct opcode group1[] = {
3541 I(Lock | PageTable, em_or),
3544 I(Lock | PageTable, em_and),
3550 static struct opcode group1A[] = {
3551 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3554 static struct opcode group3[] = {
3555 I(DstMem | SrcImm, em_test),
3556 I(DstMem | SrcImm, em_test),
3557 I(DstMem | SrcNone | Lock, em_not),
3558 I(DstMem | SrcNone | Lock, em_neg),
3559 I(SrcMem, em_mul_ex),
3560 I(SrcMem, em_imul_ex),
3561 I(SrcMem, em_div_ex),
3562 I(SrcMem, em_idiv_ex),
3565 static struct opcode group4[] = {
3566 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3567 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3571 static struct opcode group5[] = {
3572 I(DstMem | SrcNone | Lock, em_grp45),
3573 I(DstMem | SrcNone | Lock, em_grp45),
3574 I(SrcMem | Stack, em_grp45),
3575 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3576 I(SrcMem | Stack, em_grp45),
3577 I(SrcMemFAddr | ImplicitOps, em_grp45),
3578 I(SrcMem | Stack, em_grp45), N,
3581 static struct opcode group6[] = {
3584 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3585 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3589 static struct group_dual group7 = { {
3590 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3591 II(Mov | DstMem | Priv, em_sidt, sidt),
3592 II(SrcMem | Priv, em_lgdt, lgdt),
3593 II(SrcMem | Priv, em_lidt, lidt),
3594 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3595 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3596 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3598 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3600 N, EXT(0, group7_rm3),
3601 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3602 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3606 static struct opcode group8[] = {
3608 I(DstMem | SrcImmByte, em_bt),
3609 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3610 I(DstMem | SrcImmByte | Lock, em_btr),
3611 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3614 static struct group_dual group9 = { {
3615 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3617 N, N, N, N, N, N, N, N,
3620 static struct opcode group11[] = {
3621 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3625 static struct gprefix pfx_0f_6f_0f_7f = {
3626 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3629 static struct gprefix pfx_vmovntpx = {
3630 I(0, em_mov), N, N, N,
3633 static struct opcode opcode_table[256] = {
3635 I6ALU(Lock, em_add),
3636 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3637 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3639 I6ALU(Lock | PageTable, em_or),
3640 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3643 I6ALU(Lock, em_adc),
3644 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3645 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3647 I6ALU(Lock, em_sbb),
3648 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3649 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3651 I6ALU(Lock | PageTable, em_and), N, N,
3653 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3655 I6ALU(Lock, em_xor), N, N,
3657 I6ALU(0, em_cmp), N, N,
3661 X8(I(SrcReg | Stack, em_push)),
3663 X8(I(DstReg | Stack, em_pop)),
3665 I(ImplicitOps | Stack | No64, em_pusha),
3666 I(ImplicitOps | Stack | No64, em_popa),
3667 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3670 I(SrcImm | Mov | Stack, em_push),
3671 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3672 I(SrcImmByte | Mov | Stack, em_push),
3673 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3674 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3675 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3679 G(ByteOp | DstMem | SrcImm, group1),
3680 G(DstMem | SrcImm, group1),
3681 G(ByteOp | DstMem | SrcImm | No64, group1),
3682 G(DstMem | SrcImmByte, group1),
3683 I2bv(DstMem | SrcReg | ModRM, em_test),
3684 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3686 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3687 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3688 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3689 D(ModRM | SrcMem | NoAccess | DstReg),
3690 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3693 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3695 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3696 I(SrcImmFAddr | No64, em_call_far), N,
3697 II(ImplicitOps | Stack, em_pushf, pushf),
3698 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3700 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3701 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3702 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3703 I2bv(SrcSI | DstDI | String, em_cmp),
3705 I2bv(DstAcc | SrcImm, em_test),
3706 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3707 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3708 I2bv(SrcAcc | DstDI | String, em_cmp),
3710 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3712 X8(I(DstReg | SrcImm | Mov, em_mov)),
3714 D2bv(DstMem | SrcImmByte | ModRM),
3715 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3716 I(ImplicitOps | Stack, em_ret),
3717 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3718 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3719 G(ByteOp, group11), G(0, group11),
3721 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3722 N, I(ImplicitOps | Stack, em_ret_far),
3723 D(ImplicitOps), DI(SrcImmByte, intn),
3724 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3726 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3729 N, N, N, N, N, N, N, N,
3731 X3(I(SrcImmByte, em_loop)),
3732 I(SrcImmByte, em_jcxz),
3733 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3734 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3736 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3737 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3738 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3739 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3741 N, DI(ImplicitOps, icebp), N, N,
3742 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3743 G(ByteOp, group3), G(0, group3),
3745 D(ImplicitOps), D(ImplicitOps),
3746 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3747 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3750 static struct opcode twobyte_table[256] = {
3752 G(0, group6), GD(0, &group7), N, N,
3753 N, I(ImplicitOps | VendorSpecific, em_syscall),
3754 II(ImplicitOps | Priv, em_clts, clts), N,
3755 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3756 N, D(ImplicitOps | ModRM), N, N,
3758 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3760 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3761 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3762 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3763 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3765 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3768 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3769 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3770 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3771 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3772 I(ImplicitOps | VendorSpecific, em_sysenter),
3773 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3775 N, N, N, N, N, N, N, N,
3777 X16(D(DstReg | SrcMem | ModRM | Mov)),
3779 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3784 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3789 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3793 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3795 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3796 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3797 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3798 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3800 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3801 DI(ImplicitOps, rsm),
3802 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3803 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3804 D(DstMem | SrcReg | Src2CL | ModRM),
3805 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3807 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3808 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3809 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3810 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3811 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3812 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3816 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3817 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3818 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3820 D2bv(DstMem | SrcReg | ModRM | Lock),
3821 N, D(DstMem | SrcReg | ModRM | Mov),
3822 N, N, N, GD(0, &group9),
3824 X8(I(DstReg, em_bswap)),
3826 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3828 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3830 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3847 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3851 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3857 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3858 unsigned size, bool sign_extension)
3860 int rc = X86EMUL_CONTINUE;
3864 op->addr.mem.ea = ctxt->_eip;
3865 /* NB. Immediates are sign-extended as necessary. */
3866 switch (op->bytes) {
3868 op->val = insn_fetch(s8, ctxt);
3871 op->val = insn_fetch(s16, ctxt);
3874 op->val = insn_fetch(s32, ctxt);
3877 if (!sign_extension) {
3878 switch (op->bytes) {
3886 op->val &= 0xffffffff;
3894 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3897 int rc = X86EMUL_CONTINUE;
3901 decode_register_operand(ctxt, op);
3904 rc = decode_imm(ctxt, op, 1, false);
3907 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3911 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3912 fetch_bit_operand(ctxt);
3913 op->orig_val = op->val;
3916 ctxt->memop.bytes = 8;
3920 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3921 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3922 fetch_register_operand(op);
3923 op->orig_val = op->val;
3927 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3929 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3930 op->addr.mem.seg = VCPU_SREG_ES;
3936 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3937 fetch_register_operand(op);
3941 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3944 rc = decode_imm(ctxt, op, 1, true);
3951 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3954 ctxt->memop.bytes = 1;
3957 ctxt->memop.bytes = 2;
3960 ctxt->memop.bytes = 4;
3963 rc = decode_imm(ctxt, op, 2, false);
3966 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3970 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3972 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3973 op->addr.mem.seg = seg_override(ctxt);
3978 op->addr.mem.ea = ctxt->_eip;
3979 op->bytes = ctxt->op_bytes + 2;
3980 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3983 ctxt->memop.bytes = ctxt->op_bytes + 2;
3986 op->val = VCPU_SREG_ES;
3989 op->val = VCPU_SREG_CS;
3992 op->val = VCPU_SREG_SS;
3995 op->val = VCPU_SREG_DS;
3998 op->val = VCPU_SREG_FS;
4001 op->val = VCPU_SREG_GS;
4004 /* Special instructions do their own operand decoding. */
4006 op->type = OP_NONE; /* Disable writeback. */
4014 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4016 int rc = X86EMUL_CONTINUE;
4017 int mode = ctxt->mode;
4018 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4019 bool op_prefix = false;
4020 struct opcode opcode;
4022 ctxt->memop.type = OP_NONE;
4023 ctxt->memopp = NULL;
4024 ctxt->_eip = ctxt->eip;
4025 ctxt->fetch.start = ctxt->_eip;
4026 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4028 memcpy(ctxt->fetch.data, insn, insn_len);
4031 case X86EMUL_MODE_REAL:
4032 case X86EMUL_MODE_VM86:
4033 case X86EMUL_MODE_PROT16:
4034 def_op_bytes = def_ad_bytes = 2;
4036 case X86EMUL_MODE_PROT32:
4037 def_op_bytes = def_ad_bytes = 4;
4039 #ifdef CONFIG_X86_64
4040 case X86EMUL_MODE_PROT64:
4046 return EMULATION_FAILED;
4049 ctxt->op_bytes = def_op_bytes;
4050 ctxt->ad_bytes = def_ad_bytes;
4052 /* Legacy prefixes. */
4054 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4055 case 0x66: /* operand-size override */
4057 /* switch between 2/4 bytes */
4058 ctxt->op_bytes = def_op_bytes ^ 6;
4060 case 0x67: /* address-size override */
4061 if (mode == X86EMUL_MODE_PROT64)
4062 /* switch between 4/8 bytes */
4063 ctxt->ad_bytes = def_ad_bytes ^ 12;
4065 /* switch between 2/4 bytes */
4066 ctxt->ad_bytes = def_ad_bytes ^ 6;
4068 case 0x26: /* ES override */
4069 case 0x2e: /* CS override */
4070 case 0x36: /* SS override */
4071 case 0x3e: /* DS override */
4072 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4074 case 0x64: /* FS override */
4075 case 0x65: /* GS override */
4076 set_seg_override(ctxt, ctxt->b & 7);
4078 case 0x40 ... 0x4f: /* REX */
4079 if (mode != X86EMUL_MODE_PROT64)
4081 ctxt->rex_prefix = ctxt->b;
4083 case 0xf0: /* LOCK */
4084 ctxt->lock_prefix = 1;
4086 case 0xf2: /* REPNE/REPNZ */
4087 case 0xf3: /* REP/REPE/REPZ */
4088 ctxt->rep_prefix = ctxt->b;
4094 /* Any legacy prefix after a REX prefix nullifies its effect. */
4096 ctxt->rex_prefix = 0;
4102 if (ctxt->rex_prefix & 8)
4103 ctxt->op_bytes = 8; /* REX.W */
4105 /* Opcode byte(s). */
4106 opcode = opcode_table[ctxt->b];
4107 /* Two-byte opcode? */
4108 if (ctxt->b == 0x0f) {
4110 ctxt->b = insn_fetch(u8, ctxt);
4111 opcode = twobyte_table[ctxt->b];
4113 ctxt->d = opcode.flags;
4115 if (ctxt->d & ModRM)
4116 ctxt->modrm = insn_fetch(u8, ctxt);
4118 while (ctxt->d & GroupMask) {
4119 switch (ctxt->d & GroupMask) {
4121 goffset = (ctxt->modrm >> 3) & 7;
4122 opcode = opcode.u.group[goffset];
4125 goffset = (ctxt->modrm >> 3) & 7;
4126 if ((ctxt->modrm >> 6) == 3)
4127 opcode = opcode.u.gdual->mod3[goffset];
4129 opcode = opcode.u.gdual->mod012[goffset];
4132 goffset = ctxt->modrm & 7;
4133 opcode = opcode.u.group[goffset];
4136 if (ctxt->rep_prefix && op_prefix)
4137 return EMULATION_FAILED;
4138 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4139 switch (simd_prefix) {
4140 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4141 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4142 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4143 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4147 return EMULATION_FAILED;
4150 ctxt->d &= ~(u64)GroupMask;
4151 ctxt->d |= opcode.flags;
4154 ctxt->execute = opcode.u.execute;
4155 ctxt->check_perm = opcode.check_perm;
4156 ctxt->intercept = opcode.intercept;
4159 if (ctxt->d == 0 || (ctxt->d & Undefined))
4160 return EMULATION_FAILED;
4162 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4163 return EMULATION_FAILED;
4165 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4168 if (ctxt->d & Op3264) {
4169 if (mode == X86EMUL_MODE_PROT64)
4176 ctxt->op_bytes = 16;
4177 else if (ctxt->d & Mmx)
4180 /* ModRM and SIB bytes. */
4181 if (ctxt->d & ModRM) {
4182 rc = decode_modrm(ctxt, &ctxt->memop);
4183 if (!ctxt->has_seg_override)
4184 set_seg_override(ctxt, ctxt->modrm_seg);
4185 } else if (ctxt->d & MemAbs)
4186 rc = decode_abs(ctxt, &ctxt->memop);
4187 if (rc != X86EMUL_CONTINUE)
4190 if (!ctxt->has_seg_override)
4191 set_seg_override(ctxt, VCPU_SREG_DS);
4193 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4195 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4196 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4199 * Decode and fetch the source operand: register, memory
4202 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4203 if (rc != X86EMUL_CONTINUE)
4207 * Decode and fetch the second source operand: register, memory
4210 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4211 if (rc != X86EMUL_CONTINUE)
4214 /* Decode and fetch the destination operand: register or memory. */
4215 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4218 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4219 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4221 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4224 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4226 return ctxt->d & PageTable;
4229 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4231 /* The second termination condition only applies for REPE
4232 * and REPNE. Test if the repeat string operation prefix is
4233 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4234 * corresponding termination condition according to:
4235 * - if REPE/REPZ and ZF = 0 then done
4236 * - if REPNE/REPNZ and ZF = 1 then done
4238 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4239 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4240 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4241 ((ctxt->eflags & EFLG_ZF) == 0))
4242 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4243 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4249 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4253 ctxt->ops->get_fpu(ctxt);
4254 asm volatile("1: fwait \n\t"
4256 ".pushsection .fixup,\"ax\" \n\t"
4258 "movb $1, %[fault] \n\t"
4261 _ASM_EXTABLE(1b, 3b)
4262 : [fault]"+qm"(fault));
4263 ctxt->ops->put_fpu(ctxt);
4265 if (unlikely(fault))
4266 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4268 return X86EMUL_CONTINUE;
4271 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4274 if (op->type == OP_MM)
4275 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4278 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4280 struct x86_emulate_ops *ops = ctxt->ops;
4281 int rc = X86EMUL_CONTINUE;
4282 int saved_dst_type = ctxt->dst.type;
4284 ctxt->mem_read.pos = 0;
4286 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4287 rc = emulate_ud(ctxt);
4291 /* LOCK prefix is allowed only with some instructions */
4292 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4293 rc = emulate_ud(ctxt);
4297 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4298 rc = emulate_ud(ctxt);
4302 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4303 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4304 rc = emulate_ud(ctxt);
4308 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4309 rc = emulate_nm(ctxt);
4313 if (ctxt->d & Mmx) {
4314 rc = flush_pending_x87_faults(ctxt);
4315 if (rc != X86EMUL_CONTINUE)
4318 * Now that we know the fpu is exception safe, we can fetch
4321 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4322 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4323 if (!(ctxt->d & Mov))
4324 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4327 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4328 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4329 X86_ICPT_PRE_EXCEPT);
4330 if (rc != X86EMUL_CONTINUE)
4334 /* Privileged instruction can be executed only in CPL=0 */
4335 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4336 rc = emulate_gp(ctxt, 0);
4340 /* Instruction can only be executed in protected mode */
4341 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
4342 rc = emulate_ud(ctxt);
4346 /* Do instruction specific permission checks */
4347 if (ctxt->check_perm) {
4348 rc = ctxt->check_perm(ctxt);
4349 if (rc != X86EMUL_CONTINUE)
4353 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4354 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4355 X86_ICPT_POST_EXCEPT);
4356 if (rc != X86EMUL_CONTINUE)
4360 if (ctxt->rep_prefix && (ctxt->d & String)) {
4361 /* All REP prefixes have the same first termination condition */
4362 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
4363 ctxt->eip = ctxt->_eip;
4368 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4369 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4370 ctxt->src.valptr, ctxt->src.bytes);
4371 if (rc != X86EMUL_CONTINUE)
4373 ctxt->src.orig_val64 = ctxt->src.val64;
4376 if (ctxt->src2.type == OP_MEM) {
4377 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4378 &ctxt->src2.val, ctxt->src2.bytes);
4379 if (rc != X86EMUL_CONTINUE)
4383 if ((ctxt->d & DstMask) == ImplicitOps)
4387 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4388 /* optimisation - avoid slow emulated read if Mov */
4389 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4390 &ctxt->dst.val, ctxt->dst.bytes);
4391 if (rc != X86EMUL_CONTINUE)
4394 ctxt->dst.orig_val = ctxt->dst.val;
4398 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4399 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4400 X86_ICPT_POST_MEMACCESS);
4401 if (rc != X86EMUL_CONTINUE)
4405 if (ctxt->execute) {
4406 rc = ctxt->execute(ctxt);
4407 if (rc != X86EMUL_CONTINUE)
4416 case 0x40 ... 0x47: /* inc r16/r32 */
4417 emulate_1op(ctxt, "inc");
4419 case 0x48 ... 0x4f: /* dec r16/r32 */
4420 emulate_1op(ctxt, "dec");
4422 case 0x63: /* movsxd */
4423 if (ctxt->mode != X86EMUL_MODE_PROT64)
4424 goto cannot_emulate;
4425 ctxt->dst.val = (s32) ctxt->src.val;
4427 case 0x70 ... 0x7f: /* jcc (short) */
4428 if (test_cc(ctxt->b, ctxt->eflags))
4429 jmp_rel(ctxt, ctxt->src.val);
4431 case 0x8d: /* lea r16/r32, m */
4432 ctxt->dst.val = ctxt->src.addr.mem.ea;
4434 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4435 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4439 case 0x98: /* cbw/cwde/cdqe */
4440 switch (ctxt->op_bytes) {
4441 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4442 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4443 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4449 case 0xcc: /* int3 */
4450 rc = emulate_int(ctxt, 3);
4452 case 0xcd: /* int n */
4453 rc = emulate_int(ctxt, ctxt->src.val);
4455 case 0xce: /* into */
4456 if (ctxt->eflags & EFLG_OF)
4457 rc = emulate_int(ctxt, 4);
4459 case 0xd0 ... 0xd1: /* Grp2 */
4462 case 0xd2 ... 0xd3: /* Grp2 */
4463 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4466 case 0xe9: /* jmp rel */
4467 case 0xeb: /* jmp rel short */
4468 jmp_rel(ctxt, ctxt->src.val);
4469 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4471 case 0xf4: /* hlt */
4472 ctxt->ops->halt(ctxt);
4474 case 0xf5: /* cmc */
4475 /* complement carry flag from eflags reg */
4476 ctxt->eflags ^= EFLG_CF;
4478 case 0xf8: /* clc */
4479 ctxt->eflags &= ~EFLG_CF;
4481 case 0xf9: /* stc */
4482 ctxt->eflags |= EFLG_CF;
4484 case 0xfc: /* cld */
4485 ctxt->eflags &= ~EFLG_DF;
4487 case 0xfd: /* std */
4488 ctxt->eflags |= EFLG_DF;
4491 goto cannot_emulate;
4494 if (rc != X86EMUL_CONTINUE)
4498 rc = writeback(ctxt);
4499 if (rc != X86EMUL_CONTINUE)
4503 * restore dst type in case the decoding will be reused
4504 * (happens for string instruction )
4506 ctxt->dst.type = saved_dst_type;
4508 if ((ctxt->d & SrcMask) == SrcSI)
4509 string_addr_inc(ctxt, seg_override(ctxt),
4510 VCPU_REGS_RSI, &ctxt->src);
4512 if ((ctxt->d & DstMask) == DstDI)
4513 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4516 if (ctxt->rep_prefix && (ctxt->d & String)) {
4517 struct read_cache *r = &ctxt->io_read;
4518 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4520 if (!string_insn_completed(ctxt)) {
4522 * Re-enter guest when pio read ahead buffer is empty
4523 * or, if it is not used, after each 1024 iteration.
4525 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4526 (r->end == 0 || r->end != r->pos)) {
4528 * Reset read cache. Usually happens before
4529 * decode, but since instruction is restarted
4530 * we have to do it here.
4532 ctxt->mem_read.end = 0;
4533 return EMULATION_RESTART;
4535 goto done; /* skip rip writeback */
4539 ctxt->eip = ctxt->_eip;
4542 if (rc == X86EMUL_PROPAGATE_FAULT)
4543 ctxt->have_exception = true;
4544 if (rc == X86EMUL_INTERCEPTED)
4545 return EMULATION_INTERCEPTED;
4547 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4551 case 0x09: /* wbinvd */
4552 (ctxt->ops->wbinvd)(ctxt);
4554 case 0x08: /* invd */
4555 case 0x0d: /* GrpP (prefetch) */
4556 case 0x18: /* Grp16 (prefetch/nop) */
4558 case 0x20: /* mov cr, reg */
4559 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4561 case 0x21: /* mov from dr to reg */
4562 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4564 case 0x40 ... 0x4f: /* cmov */
4565 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4566 if (!test_cc(ctxt->b, ctxt->eflags))
4567 ctxt->dst.type = OP_NONE; /* no writeback */
4569 case 0x80 ... 0x8f: /* jnz rel, etc*/
4570 if (test_cc(ctxt->b, ctxt->eflags))
4571 jmp_rel(ctxt, ctxt->src.val);
4573 case 0x90 ... 0x9f: /* setcc r/m8 */
4574 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4576 case 0xa4: /* shld imm8, r, r/m */
4577 case 0xa5: /* shld cl, r, r/m */
4578 emulate_2op_cl(ctxt, "shld");
4580 case 0xac: /* shrd imm8, r, r/m */
4581 case 0xad: /* shrd cl, r, r/m */
4582 emulate_2op_cl(ctxt, "shrd");
4584 case 0xae: /* clflush */
4586 case 0xb6 ... 0xb7: /* movzx */
4587 ctxt->dst.bytes = ctxt->op_bytes;
4588 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4589 : (u16) ctxt->src.val;
4591 case 0xbe ... 0xbf: /* movsx */
4592 ctxt->dst.bytes = ctxt->op_bytes;
4593 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4594 (s16) ctxt->src.val;
4596 case 0xc0 ... 0xc1: /* xadd */
4597 emulate_2op_SrcV(ctxt, "add");
4598 /* Write back the register source. */
4599 ctxt->src.val = ctxt->dst.orig_val;
4600 write_register_operand(&ctxt->src);
4602 case 0xc3: /* movnti */
4603 ctxt->dst.bytes = ctxt->op_bytes;
4604 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4605 (u64) ctxt->src.val;
4608 goto cannot_emulate;
4611 if (rc != X86EMUL_CONTINUE)
4617 return EMULATION_FAILED;