1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
145 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
146 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
147 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
148 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
150 #define X2(x...) x, x
151 #define X3(x...) X2(x), x
152 #define X4(x...) X2(x), X2(x)
153 #define X5(x...) X4(x), x
154 #define X6(x...) X4(x), X2(x)
155 #define X7(x...) X4(x), X3(x)
156 #define X8(x...) X4(x), X4(x)
157 #define X16(x...) X8(x), X8(x)
163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 const struct opcode *group;
165 const struct group_dual *gdual;
166 const struct gprefix *gprefix;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
172 struct opcode mod012[8];
173 struct opcode mod3[8];
177 struct opcode pfx_no;
178 struct opcode pfx_66;
179 struct opcode pfx_f2;
180 struct opcode pfx_f3;
183 /* EFLAGS bit definitions. */
184 #define EFLG_ID (1<<21)
185 #define EFLG_VIP (1<<20)
186 #define EFLG_VIF (1<<19)
187 #define EFLG_AC (1<<18)
188 #define EFLG_VM (1<<17)
189 #define EFLG_RF (1<<16)
190 #define EFLG_IOPL (3<<12)
191 #define EFLG_NT (1<<14)
192 #define EFLG_OF (1<<11)
193 #define EFLG_DF (1<<10)
194 #define EFLG_IF (1<<9)
195 #define EFLG_TF (1<<8)
196 #define EFLG_SF (1<<7)
197 #define EFLG_ZF (1<<6)
198 #define EFLG_AF (1<<4)
199 #define EFLG_PF (1<<2)
200 #define EFLG_CF (1<<0)
202 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203 #define EFLG_RESERVED_ONE_MASK 2
205 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
207 if (!(ctxt->regs_valid & (1 << nr))) {
208 ctxt->regs_valid |= 1 << nr;
209 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
211 return ctxt->_regs[nr];
214 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
216 ctxt->regs_valid |= 1 << nr;
217 ctxt->regs_dirty |= 1 << nr;
218 return &ctxt->_regs[nr];
221 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
224 return reg_write(ctxt, nr);
227 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
231 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
232 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
235 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
237 ctxt->regs_dirty = 0;
238 ctxt->regs_valid = 0;
242 * Instruction emulation:
243 * Most instructions are emulated directly via a fragment of inline assembly
244 * code. This allows us to save/restore EFLAGS and thus very easily pick up
245 * any modified flags.
248 #if defined(CONFIG_X86_64)
249 #define _LO32 "k" /* force 32-bit operand */
250 #define _STK "%%rsp" /* stack pointer */
251 #elif defined(__i386__)
252 #define _LO32 "" /* force 32-bit operand */
253 #define _STK "%%esp" /* stack pointer */
257 * These EFLAGS bits are restored from saved value during emulation, and
258 * any changes are written back to the saved value after emulation.
260 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
262 /* Before executing instruction: restore necessary bits in EFLAGS. */
263 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
264 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
265 "movl %"_sav",%"_LO32 _tmp"; " \
268 "movl %"_msk",%"_LO32 _tmp"; " \
269 "andl %"_LO32 _tmp",("_STK"); " \
271 "notl %"_LO32 _tmp"; " \
272 "andl %"_LO32 _tmp",("_STK"); " \
273 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
275 "orl %"_LO32 _tmp",("_STK"); " \
279 /* After executing instruction: write-back necessary bits in EFLAGS. */
280 #define _POST_EFLAGS(_sav, _msk, _tmp) \
281 /* _sav |= EFLAGS & _msk; */ \
284 "andl %"_msk",%"_LO32 _tmp"; " \
285 "orl %"_LO32 _tmp",%"_sav"; "
293 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
295 __asm__ __volatile__ ( \
296 _PRE_EFLAGS("0", "4", "2") \
297 _op _suffix " %"_x"3,%1; " \
298 _POST_EFLAGS("0", "4", "2") \
299 : "=m" ((ctxt)->eflags), \
300 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
302 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
306 /* Raw emulation: instruction has two explicit operands. */
307 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
309 unsigned long _tmp; \
311 switch ((ctxt)->dst.bytes) { \
313 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
316 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
319 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
324 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
326 unsigned long _tmp; \
327 switch ((ctxt)->dst.bytes) { \
329 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
332 __emulate_2op_nobyte(ctxt, _op, \
333 _wx, _wy, _lx, _ly, _qx, _qy); \
338 /* Source operand is byte-sized and may be restricted to just %cl. */
339 #define emulate_2op_SrcB(ctxt, _op) \
340 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
342 /* Source operand is byte, word, long or quad sized. */
343 #define emulate_2op_SrcV(ctxt, _op) \
344 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
346 /* Source operand is word, long or quad sized. */
347 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
348 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
350 /* Instruction has three operands and one operand is stored in ECX register */
351 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
353 unsigned long _tmp; \
354 _type _clv = (ctxt)->src2.val; \
355 _type _srcv = (ctxt)->src.val; \
356 _type _dstv = (ctxt)->dst.val; \
358 __asm__ __volatile__ ( \
359 _PRE_EFLAGS("0", "5", "2") \
360 _op _suffix " %4,%1 \n" \
361 _POST_EFLAGS("0", "5", "2") \
362 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
363 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
366 (ctxt)->src2.val = (unsigned long) _clv; \
367 (ctxt)->src2.val = (unsigned long) _srcv; \
368 (ctxt)->dst.val = (unsigned long) _dstv; \
371 #define emulate_2op_cl(ctxt, _op) \
373 switch ((ctxt)->dst.bytes) { \
375 __emulate_2op_cl(ctxt, _op, "w", u16); \
378 __emulate_2op_cl(ctxt, _op, "l", u32); \
381 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
386 #define __emulate_1op(ctxt, _op, _suffix) \
388 unsigned long _tmp; \
390 __asm__ __volatile__ ( \
391 _PRE_EFLAGS("0", "3", "2") \
392 _op _suffix " %1; " \
393 _POST_EFLAGS("0", "3", "2") \
394 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
396 : "i" (EFLAGS_MASK)); \
399 /* Instruction has only one explicit operand (no source operand). */
400 #define emulate_1op(ctxt, _op) \
402 switch ((ctxt)->dst.bytes) { \
403 case 1: __emulate_1op(ctxt, _op, "b"); break; \
404 case 2: __emulate_1op(ctxt, _op, "w"); break; \
405 case 4: __emulate_1op(ctxt, _op, "l"); break; \
406 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
410 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
412 unsigned long _tmp; \
413 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
414 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
416 __asm__ __volatile__ ( \
417 _PRE_EFLAGS("0", "5", "1") \
419 _op _suffix " %6; " \
421 _POST_EFLAGS("0", "5", "1") \
422 ".pushsection .fixup,\"ax\" \n\t" \
423 "3: movb $1, %4 \n\t" \
426 _ASM_EXTABLE(1b, 3b) \
427 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
428 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
429 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
430 "a" (*rax), "d" (*rdx)); \
433 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
434 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
436 switch((ctxt)->src.bytes) { \
438 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
441 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
444 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
447 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
452 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
453 enum x86_intercept intercept,
454 enum x86_intercept_stage stage)
456 struct x86_instruction_info info = {
457 .intercept = intercept,
458 .rep_prefix = ctxt->rep_prefix,
459 .modrm_mod = ctxt->modrm_mod,
460 .modrm_reg = ctxt->modrm_reg,
461 .modrm_rm = ctxt->modrm_rm,
462 .src_val = ctxt->src.val64,
463 .src_bytes = ctxt->src.bytes,
464 .dst_bytes = ctxt->dst.bytes,
465 .ad_bytes = ctxt->ad_bytes,
466 .next_rip = ctxt->eip,
469 return ctxt->ops->intercept(ctxt, &info, stage);
472 static void assign_masked(ulong *dest, ulong src, ulong mask)
474 *dest = (*dest & ~mask) | (src & mask);
477 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
479 return (1UL << (ctxt->ad_bytes << 3)) - 1;
482 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
485 struct desc_struct ss;
487 if (ctxt->mode == X86EMUL_MODE_PROT64)
489 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
490 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
493 static int stack_size(struct x86_emulate_ctxt *ctxt)
495 return (__fls(stack_mask(ctxt)) + 1) >> 3;
498 /* Access/update address held in a register, based on addressing mode. */
499 static inline unsigned long
500 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
502 if (ctxt->ad_bytes == sizeof(unsigned long))
505 return reg & ad_mask(ctxt);
508 static inline unsigned long
509 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
511 return address_mask(ctxt, reg);
514 static void masked_increment(ulong *reg, ulong mask, int inc)
516 assign_masked(reg, *reg + inc, mask);
520 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
524 if (ctxt->ad_bytes == sizeof(unsigned long))
527 mask = ad_mask(ctxt);
528 masked_increment(reg, mask, inc);
531 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
533 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
536 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
538 register_address_increment(ctxt, &ctxt->_eip, rel);
541 static u32 desc_limit_scaled(struct desc_struct *desc)
543 u32 limit = get_desc_limit(desc);
545 return desc->g ? (limit << 12) | 0xfff : limit;
548 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
550 ctxt->has_seg_override = true;
551 ctxt->seg_override = seg;
554 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
556 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
559 return ctxt->ops->get_cached_segment_base(ctxt, seg);
562 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
564 if (!ctxt->has_seg_override)
567 return ctxt->seg_override;
570 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
571 u32 error, bool valid)
573 ctxt->exception.vector = vec;
574 ctxt->exception.error_code = error;
575 ctxt->exception.error_code_valid = valid;
576 return X86EMUL_PROPAGATE_FAULT;
579 static int emulate_db(struct x86_emulate_ctxt *ctxt)
581 return emulate_exception(ctxt, DB_VECTOR, 0, false);
584 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
586 return emulate_exception(ctxt, GP_VECTOR, err, true);
589 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
591 return emulate_exception(ctxt, SS_VECTOR, err, true);
594 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
596 return emulate_exception(ctxt, UD_VECTOR, 0, false);
599 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
601 return emulate_exception(ctxt, TS_VECTOR, err, true);
604 static int emulate_de(struct x86_emulate_ctxt *ctxt)
606 return emulate_exception(ctxt, DE_VECTOR, 0, false);
609 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
611 return emulate_exception(ctxt, NM_VECTOR, 0, false);
614 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
617 struct desc_struct desc;
619 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
623 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
628 struct desc_struct desc;
630 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
631 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
635 * x86 defines three classes of vector instructions: explicitly
636 * aligned, explicitly unaligned, and the rest, which change behaviour
637 * depending on whether they're AVX encoded or not.
639 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
640 * subject to the same check.
642 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
644 if (likely(size < 16))
647 if (ctxt->d & Aligned)
649 else if (ctxt->d & Unaligned)
651 else if (ctxt->d & Avx)
657 static int __linearize(struct x86_emulate_ctxt *ctxt,
658 struct segmented_address addr,
659 unsigned size, bool write, bool fetch,
662 struct desc_struct desc;
669 la = seg_base(ctxt, addr.seg) + addr.ea;
670 switch (ctxt->mode) {
671 case X86EMUL_MODE_PROT64:
672 if (((signed long)la << 16) >> 16 != la)
673 return emulate_gp(ctxt, 0);
676 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
680 /* code segment or read-only data segment */
681 if (((desc.type & 8) || !(desc.type & 2)) && write)
683 /* unreadable code segment */
684 if (!fetch && (desc.type & 8) && !(desc.type & 2))
686 lim = desc_limit_scaled(&desc);
687 if ((desc.type & 8) || !(desc.type & 4)) {
688 /* expand-up segment */
689 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
692 /* expand-down segment */
693 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
695 lim = desc.d ? 0xffffffff : 0xffff;
696 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
699 cpl = ctxt->ops->cpl(ctxt);
700 if (ctxt->mode == X86EMUL_MODE_REAL)
705 if (!(desc.type & 8)) {
709 } else if ((desc.type & 8) && !(desc.type & 4)) {
710 /* nonconforming code segment */
713 } else if ((desc.type & 8) && (desc.type & 4)) {
714 /* conforming code segment */
720 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
722 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
723 return emulate_gp(ctxt, 0);
725 return X86EMUL_CONTINUE;
727 if (addr.seg == VCPU_SREG_SS)
728 return emulate_ss(ctxt, sel);
730 return emulate_gp(ctxt, sel);
733 static int linearize(struct x86_emulate_ctxt *ctxt,
734 struct segmented_address addr,
735 unsigned size, bool write,
738 return __linearize(ctxt, addr, size, write, false, linear);
742 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
743 struct segmented_address addr,
750 rc = linearize(ctxt, addr, size, false, &linear);
751 if (rc != X86EMUL_CONTINUE)
753 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
757 * Fetch the next byte of the instruction being emulated which is pointed to
758 * by ctxt->_eip, then increment ctxt->_eip.
760 * Also prefetch the remaining bytes of the instruction without crossing page
761 * boundary if they are not in fetch_cache yet.
763 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
765 struct fetch_cache *fc = &ctxt->fetch;
769 if (ctxt->_eip == fc->end) {
770 unsigned long linear;
771 struct segmented_address addr = { .seg = VCPU_SREG_CS,
773 cur_size = fc->end - fc->start;
774 size = min(15UL - cur_size,
775 PAGE_SIZE - offset_in_page(ctxt->_eip));
776 rc = __linearize(ctxt, addr, size, false, true, &linear);
777 if (unlikely(rc != X86EMUL_CONTINUE))
779 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
780 size, &ctxt->exception);
781 if (unlikely(rc != X86EMUL_CONTINUE))
785 *dest = fc->data[ctxt->_eip - fc->start];
787 return X86EMUL_CONTINUE;
790 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
791 void *dest, unsigned size)
795 /* x86 instructions are limited to 15 bytes. */
796 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
797 return X86EMUL_UNHANDLEABLE;
799 rc = do_insn_fetch_byte(ctxt, dest++);
800 if (rc != X86EMUL_CONTINUE)
803 return X86EMUL_CONTINUE;
806 /* Fetch next part of the instruction being emulated. */
807 #define insn_fetch(_type, _ctxt) \
808 ({ unsigned long _x; \
809 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
810 if (rc != X86EMUL_CONTINUE) \
815 #define insn_fetch_arr(_arr, _size, _ctxt) \
816 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
817 if (rc != X86EMUL_CONTINUE) \
822 * Given the 'reg' portion of a ModRM byte, and a register block, return a
823 * pointer into the block that addresses the relevant register.
824 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
826 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
831 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
832 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
834 p = reg_rmw(ctxt, modrm_reg);
838 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
839 struct segmented_address addr,
840 u16 *size, unsigned long *address, int op_bytes)
847 rc = segmented_read_std(ctxt, addr, size, 2);
848 if (rc != X86EMUL_CONTINUE)
851 rc = segmented_read_std(ctxt, addr, address, op_bytes);
855 static int test_cc(unsigned int condition, unsigned int flags)
859 switch ((condition & 15) >> 1) {
861 rc |= (flags & EFLG_OF);
863 case 1: /* b/c/nae */
864 rc |= (flags & EFLG_CF);
867 rc |= (flags & EFLG_ZF);
870 rc |= (flags & (EFLG_CF|EFLG_ZF));
873 rc |= (flags & EFLG_SF);
876 rc |= (flags & EFLG_PF);
879 rc |= (flags & EFLG_ZF);
882 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
886 /* Odd condition identifiers (lsb == 1) have inverted sense. */
887 return (!!rc ^ (condition & 1));
890 static void fetch_register_operand(struct operand *op)
894 op->val = *(u8 *)op->addr.reg;
897 op->val = *(u16 *)op->addr.reg;
900 op->val = *(u32 *)op->addr.reg;
903 op->val = *(u64 *)op->addr.reg;
908 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
910 ctxt->ops->get_fpu(ctxt);
912 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
913 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
914 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
915 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
916 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
917 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
918 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
919 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
921 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
922 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
923 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
924 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
925 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
926 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
927 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
928 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
932 ctxt->ops->put_fpu(ctxt);
935 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
938 ctxt->ops->get_fpu(ctxt);
940 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
941 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
942 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
943 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
944 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
945 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
946 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
947 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
949 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
950 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
951 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
952 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
953 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
954 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
955 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
956 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
960 ctxt->ops->put_fpu(ctxt);
963 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
965 ctxt->ops->get_fpu(ctxt);
967 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
968 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
969 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
970 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
971 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
972 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
973 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
974 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
977 ctxt->ops->put_fpu(ctxt);
980 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
982 ctxt->ops->get_fpu(ctxt);
984 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
985 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
986 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
987 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
988 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
989 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
990 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
991 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
994 ctxt->ops->put_fpu(ctxt);
997 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1000 unsigned reg = ctxt->modrm_reg;
1001 int highbyte_regs = ctxt->rex_prefix == 0;
1003 if (!(ctxt->d & ModRM))
1004 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1006 if (ctxt->d & Sse) {
1010 read_sse_reg(ctxt, &op->vec_val, reg);
1013 if (ctxt->d & Mmx) {
1022 if (ctxt->d & ByteOp) {
1023 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1026 op->addr.reg = decode_register(ctxt, reg, 0);
1027 op->bytes = ctxt->op_bytes;
1029 fetch_register_operand(op);
1030 op->orig_val = op->val;
1033 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1035 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1036 ctxt->modrm_seg = VCPU_SREG_SS;
1039 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1043 int index_reg = 0, base_reg = 0, scale;
1044 int rc = X86EMUL_CONTINUE;
1047 if (ctxt->rex_prefix) {
1048 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1049 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1050 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1053 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1054 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1055 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1056 ctxt->modrm_seg = VCPU_SREG_DS;
1058 if (ctxt->modrm_mod == 3) {
1060 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1061 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1062 if (ctxt->d & Sse) {
1065 op->addr.xmm = ctxt->modrm_rm;
1066 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1069 if (ctxt->d & Mmx) {
1072 op->addr.xmm = ctxt->modrm_rm & 7;
1075 fetch_register_operand(op);
1081 if (ctxt->ad_bytes == 2) {
1082 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1083 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1084 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1085 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1087 /* 16-bit ModR/M decode. */
1088 switch (ctxt->modrm_mod) {
1090 if (ctxt->modrm_rm == 6)
1091 modrm_ea += insn_fetch(u16, ctxt);
1094 modrm_ea += insn_fetch(s8, ctxt);
1097 modrm_ea += insn_fetch(u16, ctxt);
1100 switch (ctxt->modrm_rm) {
1102 modrm_ea += bx + si;
1105 modrm_ea += bx + di;
1108 modrm_ea += bp + si;
1111 modrm_ea += bp + di;
1120 if (ctxt->modrm_mod != 0)
1127 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1128 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1129 ctxt->modrm_seg = VCPU_SREG_SS;
1130 modrm_ea = (u16)modrm_ea;
1132 /* 32/64-bit ModR/M decode. */
1133 if ((ctxt->modrm_rm & 7) == 4) {
1134 sib = insn_fetch(u8, ctxt);
1135 index_reg |= (sib >> 3) & 7;
1136 base_reg |= sib & 7;
1139 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1140 modrm_ea += insn_fetch(s32, ctxt);
1142 modrm_ea += reg_read(ctxt, base_reg);
1143 adjust_modrm_seg(ctxt, base_reg);
1146 modrm_ea += reg_read(ctxt, index_reg) << scale;
1147 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1148 if (ctxt->mode == X86EMUL_MODE_PROT64)
1149 ctxt->rip_relative = 1;
1151 base_reg = ctxt->modrm_rm;
1152 modrm_ea += reg_read(ctxt, base_reg);
1153 adjust_modrm_seg(ctxt, base_reg);
1155 switch (ctxt->modrm_mod) {
1157 if (ctxt->modrm_rm == 5)
1158 modrm_ea += insn_fetch(s32, ctxt);
1161 modrm_ea += insn_fetch(s8, ctxt);
1164 modrm_ea += insn_fetch(s32, ctxt);
1168 op->addr.mem.ea = modrm_ea;
1173 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1176 int rc = X86EMUL_CONTINUE;
1179 switch (ctxt->ad_bytes) {
1181 op->addr.mem.ea = insn_fetch(u16, ctxt);
1184 op->addr.mem.ea = insn_fetch(u32, ctxt);
1187 op->addr.mem.ea = insn_fetch(u64, ctxt);
1194 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1198 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1199 mask = ~(ctxt->dst.bytes * 8 - 1);
1201 if (ctxt->src.bytes == 2)
1202 sv = (s16)ctxt->src.val & (s16)mask;
1203 else if (ctxt->src.bytes == 4)
1204 sv = (s32)ctxt->src.val & (s32)mask;
1206 ctxt->dst.addr.mem.ea += (sv >> 3);
1209 /* only subword offset */
1210 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1213 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1214 unsigned long addr, void *dest, unsigned size)
1217 struct read_cache *mc = &ctxt->mem_read;
1219 if (mc->pos < mc->end)
1222 WARN_ON((mc->end + size) >= sizeof(mc->data));
1224 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1226 if (rc != X86EMUL_CONTINUE)
1232 memcpy(dest, mc->data + mc->pos, size);
1234 return X86EMUL_CONTINUE;
1237 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1238 struct segmented_address addr,
1245 rc = linearize(ctxt, addr, size, false, &linear);
1246 if (rc != X86EMUL_CONTINUE)
1248 return read_emulated(ctxt, linear, data, size);
1251 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1252 struct segmented_address addr,
1259 rc = linearize(ctxt, addr, size, true, &linear);
1260 if (rc != X86EMUL_CONTINUE)
1262 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1266 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1267 struct segmented_address addr,
1268 const void *orig_data, const void *data,
1274 rc = linearize(ctxt, addr, size, true, &linear);
1275 if (rc != X86EMUL_CONTINUE)
1277 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1278 size, &ctxt->exception);
1281 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1282 unsigned int size, unsigned short port,
1285 struct read_cache *rc = &ctxt->io_read;
1287 if (rc->pos == rc->end) { /* refill pio read ahead */
1288 unsigned int in_page, n;
1289 unsigned int count = ctxt->rep_prefix ?
1290 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1291 in_page = (ctxt->eflags & EFLG_DF) ?
1292 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1293 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1294 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1298 rc->pos = rc->end = 0;
1299 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1304 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1305 ctxt->dst.data = rc->data + rc->pos;
1306 ctxt->dst.type = OP_MEM_STR;
1307 ctxt->dst.count = (rc->end - rc->pos) / size;
1310 memcpy(dest, rc->data + rc->pos, size);
1316 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1317 u16 index, struct desc_struct *desc)
1322 ctxt->ops->get_idt(ctxt, &dt);
1324 if (dt.size < index * 8 + 7)
1325 return emulate_gp(ctxt, index << 3 | 0x2);
1327 addr = dt.address + index * 8;
1328 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1332 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1333 u16 selector, struct desc_ptr *dt)
1335 const struct x86_emulate_ops *ops = ctxt->ops;
1337 if (selector & 1 << 2) {
1338 struct desc_struct desc;
1341 memset (dt, 0, sizeof *dt);
1342 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1345 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1346 dt->address = get_desc_base(&desc);
1348 ops->get_gdt(ctxt, dt);
1351 /* allowed just for 8 bytes segments */
1352 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1353 u16 selector, struct desc_struct *desc,
1357 u16 index = selector >> 3;
1360 get_descriptor_table_ptr(ctxt, selector, &dt);
1362 if (dt.size < index * 8 + 7)
1363 return emulate_gp(ctxt, selector & 0xfffc);
1365 *desc_addr_p = addr = dt.address + index * 8;
1366 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1370 /* allowed just for 8 bytes segments */
1371 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1372 u16 selector, struct desc_struct *desc)
1375 u16 index = selector >> 3;
1378 get_descriptor_table_ptr(ctxt, selector, &dt);
1380 if (dt.size < index * 8 + 7)
1381 return emulate_gp(ctxt, selector & 0xfffc);
1383 addr = dt.address + index * 8;
1384 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1388 /* Does not support long mode */
1389 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1390 u16 selector, int seg)
1392 struct desc_struct seg_desc, old_desc;
1394 unsigned err_vec = GP_VECTOR;
1396 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1401 memset(&seg_desc, 0, sizeof seg_desc);
1403 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1404 || ctxt->mode == X86EMUL_MODE_REAL) {
1405 /* set real mode segment descriptor */
1406 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1407 set_desc_base(&seg_desc, selector << 4);
1412 cpl = ctxt->ops->cpl(ctxt);
1414 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1415 if ((seg == VCPU_SREG_CS
1416 || (seg == VCPU_SREG_SS
1417 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1418 || seg == VCPU_SREG_TR)
1422 /* TR should be in GDT only */
1423 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1426 if (null_selector) /* for NULL selector skip all following checks */
1429 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1430 if (ret != X86EMUL_CONTINUE)
1433 err_code = selector & 0xfffc;
1434 err_vec = GP_VECTOR;
1436 /* can't load system descriptor into segment selector */
1437 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1441 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1450 * segment is not a writable data segment or segment
1451 * selector's RPL != CPL or segment selector's RPL != CPL
1453 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1457 if (!(seg_desc.type & 8))
1460 if (seg_desc.type & 4) {
1466 if (rpl > cpl || dpl != cpl)
1469 /* CS(RPL) <- CPL */
1470 selector = (selector & 0xfffc) | cpl;
1473 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1475 old_desc = seg_desc;
1476 seg_desc.type |= 2; /* busy */
1477 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1478 sizeof(seg_desc), &ctxt->exception);
1479 if (ret != X86EMUL_CONTINUE)
1482 case VCPU_SREG_LDTR:
1483 if (seg_desc.s || seg_desc.type != 2)
1486 default: /* DS, ES, FS, or GS */
1488 * segment is not a data or readable code segment or
1489 * ((segment is a data or nonconforming code segment)
1490 * and (both RPL and CPL > DPL))
1492 if ((seg_desc.type & 0xa) == 0x8 ||
1493 (((seg_desc.type & 0xc) != 0xc) &&
1494 (rpl > dpl && cpl > dpl)))
1500 /* mark segment as accessed */
1502 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1503 if (ret != X86EMUL_CONTINUE)
1507 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1508 return X86EMUL_CONTINUE;
1510 emulate_exception(ctxt, err_vec, err_code, true);
1511 return X86EMUL_PROPAGATE_FAULT;
1514 static void write_register_operand(struct operand *op)
1516 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1517 switch (op->bytes) {
1519 *(u8 *)op->addr.reg = (u8)op->val;
1522 *(u16 *)op->addr.reg = (u16)op->val;
1525 *op->addr.reg = (u32)op->val;
1526 break; /* 64b: zero-extend */
1528 *op->addr.reg = op->val;
1533 static int writeback(struct x86_emulate_ctxt *ctxt)
1537 switch (ctxt->dst.type) {
1539 write_register_operand(&ctxt->dst);
1542 if (ctxt->lock_prefix)
1543 rc = segmented_cmpxchg(ctxt,
1545 &ctxt->dst.orig_val,
1549 rc = segmented_write(ctxt,
1553 if (rc != X86EMUL_CONTINUE)
1557 rc = segmented_write(ctxt,
1560 ctxt->dst.bytes * ctxt->dst.count);
1561 if (rc != X86EMUL_CONTINUE)
1565 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1568 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1576 return X86EMUL_CONTINUE;
1579 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1581 struct segmented_address addr;
1583 rsp_increment(ctxt, -bytes);
1584 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1585 addr.seg = VCPU_SREG_SS;
1587 return segmented_write(ctxt, addr, data, bytes);
1590 static int em_push(struct x86_emulate_ctxt *ctxt)
1592 /* Disable writeback. */
1593 ctxt->dst.type = OP_NONE;
1594 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1597 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1598 void *dest, int len)
1601 struct segmented_address addr;
1603 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1604 addr.seg = VCPU_SREG_SS;
1605 rc = segmented_read(ctxt, addr, dest, len);
1606 if (rc != X86EMUL_CONTINUE)
1609 rsp_increment(ctxt, len);
1613 static int em_pop(struct x86_emulate_ctxt *ctxt)
1615 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1618 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1619 void *dest, int len)
1622 unsigned long val, change_mask;
1623 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1624 int cpl = ctxt->ops->cpl(ctxt);
1626 rc = emulate_pop(ctxt, &val, len);
1627 if (rc != X86EMUL_CONTINUE)
1630 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1631 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1633 switch(ctxt->mode) {
1634 case X86EMUL_MODE_PROT64:
1635 case X86EMUL_MODE_PROT32:
1636 case X86EMUL_MODE_PROT16:
1638 change_mask |= EFLG_IOPL;
1640 change_mask |= EFLG_IF;
1642 case X86EMUL_MODE_VM86:
1644 return emulate_gp(ctxt, 0);
1645 change_mask |= EFLG_IF;
1647 default: /* real mode */
1648 change_mask |= (EFLG_IOPL | EFLG_IF);
1652 *(unsigned long *)dest =
1653 (ctxt->eflags & ~change_mask) | (val & change_mask);
1658 static int em_popf(struct x86_emulate_ctxt *ctxt)
1660 ctxt->dst.type = OP_REG;
1661 ctxt->dst.addr.reg = &ctxt->eflags;
1662 ctxt->dst.bytes = ctxt->op_bytes;
1663 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1666 static int em_enter(struct x86_emulate_ctxt *ctxt)
1669 unsigned frame_size = ctxt->src.val;
1670 unsigned nesting_level = ctxt->src2.val & 31;
1674 return X86EMUL_UNHANDLEABLE;
1676 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1677 rc = push(ctxt, &rbp, stack_size(ctxt));
1678 if (rc != X86EMUL_CONTINUE)
1680 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1682 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1683 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1685 return X86EMUL_CONTINUE;
1688 static int em_leave(struct x86_emulate_ctxt *ctxt)
1690 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1692 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1695 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1697 int seg = ctxt->src2.val;
1699 ctxt->src.val = get_segment_selector(ctxt, seg);
1701 return em_push(ctxt);
1704 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1706 int seg = ctxt->src2.val;
1707 unsigned long selector;
1710 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1711 if (rc != X86EMUL_CONTINUE)
1714 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1718 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1720 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1721 int rc = X86EMUL_CONTINUE;
1722 int reg = VCPU_REGS_RAX;
1724 while (reg <= VCPU_REGS_RDI) {
1725 (reg == VCPU_REGS_RSP) ?
1726 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1729 if (rc != X86EMUL_CONTINUE)
1738 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1740 ctxt->src.val = (unsigned long)ctxt->eflags;
1741 return em_push(ctxt);
1744 static int em_popa(struct x86_emulate_ctxt *ctxt)
1746 int rc = X86EMUL_CONTINUE;
1747 int reg = VCPU_REGS_RDI;
1749 while (reg >= VCPU_REGS_RAX) {
1750 if (reg == VCPU_REGS_RSP) {
1751 rsp_increment(ctxt, ctxt->op_bytes);
1755 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1756 if (rc != X86EMUL_CONTINUE)
1763 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1765 const struct x86_emulate_ops *ops = ctxt->ops;
1772 /* TODO: Add limit checks */
1773 ctxt->src.val = ctxt->eflags;
1775 if (rc != X86EMUL_CONTINUE)
1778 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1780 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1782 if (rc != X86EMUL_CONTINUE)
1785 ctxt->src.val = ctxt->_eip;
1787 if (rc != X86EMUL_CONTINUE)
1790 ops->get_idt(ctxt, &dt);
1792 eip_addr = dt.address + (irq << 2);
1793 cs_addr = dt.address + (irq << 2) + 2;
1795 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1796 if (rc != X86EMUL_CONTINUE)
1799 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1800 if (rc != X86EMUL_CONTINUE)
1803 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1804 if (rc != X86EMUL_CONTINUE)
1812 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1816 invalidate_registers(ctxt);
1817 rc = __emulate_int_real(ctxt, irq);
1818 if (rc == X86EMUL_CONTINUE)
1819 writeback_registers(ctxt);
1823 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1825 switch(ctxt->mode) {
1826 case X86EMUL_MODE_REAL:
1827 return __emulate_int_real(ctxt, irq);
1828 case X86EMUL_MODE_VM86:
1829 case X86EMUL_MODE_PROT16:
1830 case X86EMUL_MODE_PROT32:
1831 case X86EMUL_MODE_PROT64:
1833 /* Protected mode interrupts unimplemented yet */
1834 return X86EMUL_UNHANDLEABLE;
1838 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1840 int rc = X86EMUL_CONTINUE;
1841 unsigned long temp_eip = 0;
1842 unsigned long temp_eflags = 0;
1843 unsigned long cs = 0;
1844 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1845 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1846 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1847 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1849 /* TODO: Add stack limit check */
1851 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1853 if (rc != X86EMUL_CONTINUE)
1856 if (temp_eip & ~0xffff)
1857 return emulate_gp(ctxt, 0);
1859 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1861 if (rc != X86EMUL_CONTINUE)
1864 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1866 if (rc != X86EMUL_CONTINUE)
1869 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1871 if (rc != X86EMUL_CONTINUE)
1874 ctxt->_eip = temp_eip;
1877 if (ctxt->op_bytes == 4)
1878 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1879 else if (ctxt->op_bytes == 2) {
1880 ctxt->eflags &= ~0xffff;
1881 ctxt->eflags |= temp_eflags;
1884 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1885 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1890 static int em_iret(struct x86_emulate_ctxt *ctxt)
1892 switch(ctxt->mode) {
1893 case X86EMUL_MODE_REAL:
1894 return emulate_iret_real(ctxt);
1895 case X86EMUL_MODE_VM86:
1896 case X86EMUL_MODE_PROT16:
1897 case X86EMUL_MODE_PROT32:
1898 case X86EMUL_MODE_PROT64:
1900 /* iret from protected mode unimplemented yet */
1901 return X86EMUL_UNHANDLEABLE;
1905 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1910 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1912 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1913 if (rc != X86EMUL_CONTINUE)
1917 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1918 return X86EMUL_CONTINUE;
1921 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1923 switch (ctxt->modrm_reg) {
1925 emulate_2op_SrcB(ctxt, "rol");
1928 emulate_2op_SrcB(ctxt, "ror");
1931 emulate_2op_SrcB(ctxt, "rcl");
1934 emulate_2op_SrcB(ctxt, "rcr");
1936 case 4: /* sal/shl */
1937 case 6: /* sal/shl */
1938 emulate_2op_SrcB(ctxt, "sal");
1941 emulate_2op_SrcB(ctxt, "shr");
1944 emulate_2op_SrcB(ctxt, "sar");
1947 return X86EMUL_CONTINUE;
1950 static int em_not(struct x86_emulate_ctxt *ctxt)
1952 ctxt->dst.val = ~ctxt->dst.val;
1953 return X86EMUL_CONTINUE;
1956 static int em_neg(struct x86_emulate_ctxt *ctxt)
1958 emulate_1op(ctxt, "neg");
1959 return X86EMUL_CONTINUE;
1962 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1966 emulate_1op_rax_rdx(ctxt, "mul", ex);
1967 return X86EMUL_CONTINUE;
1970 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1974 emulate_1op_rax_rdx(ctxt, "imul", ex);
1975 return X86EMUL_CONTINUE;
1978 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1982 emulate_1op_rax_rdx(ctxt, "div", de);
1984 return emulate_de(ctxt);
1985 return X86EMUL_CONTINUE;
1988 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1992 emulate_1op_rax_rdx(ctxt, "idiv", de);
1994 return emulate_de(ctxt);
1995 return X86EMUL_CONTINUE;
1998 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2000 int rc = X86EMUL_CONTINUE;
2002 switch (ctxt->modrm_reg) {
2004 emulate_1op(ctxt, "inc");
2007 emulate_1op(ctxt, "dec");
2009 case 2: /* call near abs */ {
2011 old_eip = ctxt->_eip;
2012 ctxt->_eip = ctxt->src.val;
2013 ctxt->src.val = old_eip;
2017 case 4: /* jmp abs */
2018 ctxt->_eip = ctxt->src.val;
2020 case 5: /* jmp far */
2021 rc = em_jmp_far(ctxt);
2030 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2032 u64 old = ctxt->dst.orig_val64;
2034 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2035 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2036 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2037 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2038 ctxt->eflags &= ~EFLG_ZF;
2040 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2041 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2043 ctxt->eflags |= EFLG_ZF;
2045 return X86EMUL_CONTINUE;
2048 static int em_ret(struct x86_emulate_ctxt *ctxt)
2050 ctxt->dst.type = OP_REG;
2051 ctxt->dst.addr.reg = &ctxt->_eip;
2052 ctxt->dst.bytes = ctxt->op_bytes;
2053 return em_pop(ctxt);
2056 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2061 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2062 if (rc != X86EMUL_CONTINUE)
2064 if (ctxt->op_bytes == 4)
2065 ctxt->_eip = (u32)ctxt->_eip;
2066 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2067 if (rc != X86EMUL_CONTINUE)
2069 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2073 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2075 /* Save real source value, then compare EAX against destination. */
2076 ctxt->src.orig_val = ctxt->src.val;
2077 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2078 emulate_2op_SrcV(ctxt, "cmp");
2080 if (ctxt->eflags & EFLG_ZF) {
2081 /* Success: write back to memory. */
2082 ctxt->dst.val = ctxt->src.orig_val;
2084 /* Failure: write the value we saw to EAX. */
2085 ctxt->dst.type = OP_REG;
2086 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2088 return X86EMUL_CONTINUE;
2091 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2093 int seg = ctxt->src2.val;
2097 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2099 rc = load_segment_descriptor(ctxt, sel, seg);
2100 if (rc != X86EMUL_CONTINUE)
2103 ctxt->dst.val = ctxt->src.val;
2108 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2109 struct desc_struct *cs, struct desc_struct *ss)
2111 cs->l = 0; /* will be adjusted later */
2112 set_desc_base(cs, 0); /* flat segment */
2113 cs->g = 1; /* 4kb granularity */
2114 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2115 cs->type = 0x0b; /* Read, Execute, Accessed */
2117 cs->dpl = 0; /* will be adjusted later */
2122 set_desc_base(ss, 0); /* flat segment */
2123 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2124 ss->g = 1; /* 4kb granularity */
2126 ss->type = 0x03; /* Read/Write, Accessed */
2127 ss->d = 1; /* 32bit stack segment */
2134 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2136 u32 eax, ebx, ecx, edx;
2139 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2140 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2141 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2142 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2145 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2147 const struct x86_emulate_ops *ops = ctxt->ops;
2148 u32 eax, ebx, ecx, edx;
2151 * syscall should always be enabled in longmode - so only become
2152 * vendor specific (cpuid) if other modes are active...
2154 if (ctxt->mode == X86EMUL_MODE_PROT64)
2159 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2161 * Intel ("GenuineIntel")
2162 * remark: Intel CPUs only support "syscall" in 64bit
2163 * longmode. Also an 64bit guest with a
2164 * 32bit compat-app running will #UD !! While this
2165 * behaviour can be fixed (by emulating) into AMD
2166 * response - CPUs of AMD can't behave like Intel.
2168 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2169 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2170 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2173 /* AMD ("AuthenticAMD") */
2174 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2175 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2176 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2179 /* AMD ("AMDisbetter!") */
2180 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2181 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2182 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2185 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2189 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2191 const struct x86_emulate_ops *ops = ctxt->ops;
2192 struct desc_struct cs, ss;
2197 /* syscall is not available in real mode */
2198 if (ctxt->mode == X86EMUL_MODE_REAL ||
2199 ctxt->mode == X86EMUL_MODE_VM86)
2200 return emulate_ud(ctxt);
2202 if (!(em_syscall_is_enabled(ctxt)))
2203 return emulate_ud(ctxt);
2205 ops->get_msr(ctxt, MSR_EFER, &efer);
2206 setup_syscalls_segments(ctxt, &cs, &ss);
2208 if (!(efer & EFER_SCE))
2209 return emulate_ud(ctxt);
2211 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2213 cs_sel = (u16)(msr_data & 0xfffc);
2214 ss_sel = (u16)(msr_data + 8);
2216 if (efer & EFER_LMA) {
2220 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2221 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2223 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2224 if (efer & EFER_LMA) {
2225 #ifdef CONFIG_X86_64
2226 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2229 ctxt->mode == X86EMUL_MODE_PROT64 ?
2230 MSR_LSTAR : MSR_CSTAR, &msr_data);
2231 ctxt->_eip = msr_data;
2233 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2234 ctxt->eflags &= ~(msr_data | EFLG_RF);
2238 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2239 ctxt->_eip = (u32)msr_data;
2241 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2244 return X86EMUL_CONTINUE;
2247 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2249 const struct x86_emulate_ops *ops = ctxt->ops;
2250 struct desc_struct cs, ss;
2255 ops->get_msr(ctxt, MSR_EFER, &efer);
2256 /* inject #GP if in real mode */
2257 if (ctxt->mode == X86EMUL_MODE_REAL)
2258 return emulate_gp(ctxt, 0);
2261 * Not recognized on AMD in compat mode (but is recognized in legacy
2264 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2265 && !vendor_intel(ctxt))
2266 return emulate_ud(ctxt);
2268 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2269 * Therefore, we inject an #UD.
2271 if (ctxt->mode == X86EMUL_MODE_PROT64)
2272 return emulate_ud(ctxt);
2274 setup_syscalls_segments(ctxt, &cs, &ss);
2276 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2277 switch (ctxt->mode) {
2278 case X86EMUL_MODE_PROT32:
2279 if ((msr_data & 0xfffc) == 0x0)
2280 return emulate_gp(ctxt, 0);
2282 case X86EMUL_MODE_PROT64:
2283 if (msr_data == 0x0)
2284 return emulate_gp(ctxt, 0);
2290 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2291 cs_sel = (u16)msr_data;
2292 cs_sel &= ~SELECTOR_RPL_MASK;
2293 ss_sel = cs_sel + 8;
2294 ss_sel &= ~SELECTOR_RPL_MASK;
2295 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2300 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2301 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2303 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2304 ctxt->_eip = msr_data;
2306 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2307 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2309 return X86EMUL_CONTINUE;
2312 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2314 const struct x86_emulate_ops *ops = ctxt->ops;
2315 struct desc_struct cs, ss;
2318 u16 cs_sel = 0, ss_sel = 0;
2320 /* inject #GP if in real mode or Virtual 8086 mode */
2321 if (ctxt->mode == X86EMUL_MODE_REAL ||
2322 ctxt->mode == X86EMUL_MODE_VM86)
2323 return emulate_gp(ctxt, 0);
2325 setup_syscalls_segments(ctxt, &cs, &ss);
2327 if ((ctxt->rex_prefix & 0x8) != 0x0)
2328 usermode = X86EMUL_MODE_PROT64;
2330 usermode = X86EMUL_MODE_PROT32;
2334 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2336 case X86EMUL_MODE_PROT32:
2337 cs_sel = (u16)(msr_data + 16);
2338 if ((msr_data & 0xfffc) == 0x0)
2339 return emulate_gp(ctxt, 0);
2340 ss_sel = (u16)(msr_data + 24);
2342 case X86EMUL_MODE_PROT64:
2343 cs_sel = (u16)(msr_data + 32);
2344 if (msr_data == 0x0)
2345 return emulate_gp(ctxt, 0);
2346 ss_sel = cs_sel + 8;
2351 cs_sel |= SELECTOR_RPL_MASK;
2352 ss_sel |= SELECTOR_RPL_MASK;
2354 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2355 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2357 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2358 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2360 return X86EMUL_CONTINUE;
2363 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2366 if (ctxt->mode == X86EMUL_MODE_REAL)
2368 if (ctxt->mode == X86EMUL_MODE_VM86)
2370 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2371 return ctxt->ops->cpl(ctxt) > iopl;
2374 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2377 const struct x86_emulate_ops *ops = ctxt->ops;
2378 struct desc_struct tr_seg;
2381 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2382 unsigned mask = (1 << len) - 1;
2385 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2388 if (desc_limit_scaled(&tr_seg) < 103)
2390 base = get_desc_base(&tr_seg);
2391 #ifdef CONFIG_X86_64
2392 base |= ((u64)base3) << 32;
2394 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2395 if (r != X86EMUL_CONTINUE)
2397 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2399 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2400 if (r != X86EMUL_CONTINUE)
2402 if ((perm >> bit_idx) & mask)
2407 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2413 if (emulator_bad_iopl(ctxt))
2414 if (!emulator_io_port_access_allowed(ctxt, port, len))
2417 ctxt->perm_ok = true;
2422 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2423 struct tss_segment_16 *tss)
2425 tss->ip = ctxt->_eip;
2426 tss->flag = ctxt->eflags;
2427 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2428 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2429 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2430 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2431 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2432 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2433 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2434 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2436 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2437 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2438 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2439 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2440 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2443 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2444 struct tss_segment_16 *tss)
2448 ctxt->_eip = tss->ip;
2449 ctxt->eflags = tss->flag | 2;
2450 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2451 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2452 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2453 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2454 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2455 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2456 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2457 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2460 * SDM says that segment selectors are loaded before segment
2463 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2464 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2465 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2466 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2467 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2470 * Now load segment descriptors. If fault happens at this stage
2471 * it is handled in a context of new task
2473 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2474 if (ret != X86EMUL_CONTINUE)
2476 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2477 if (ret != X86EMUL_CONTINUE)
2479 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2480 if (ret != X86EMUL_CONTINUE)
2482 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2483 if (ret != X86EMUL_CONTINUE)
2485 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2486 if (ret != X86EMUL_CONTINUE)
2489 return X86EMUL_CONTINUE;
2492 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2493 u16 tss_selector, u16 old_tss_sel,
2494 ulong old_tss_base, struct desc_struct *new_desc)
2496 const struct x86_emulate_ops *ops = ctxt->ops;
2497 struct tss_segment_16 tss_seg;
2499 u32 new_tss_base = get_desc_base(new_desc);
2501 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2503 if (ret != X86EMUL_CONTINUE)
2504 /* FIXME: need to provide precise fault address */
2507 save_state_to_tss16(ctxt, &tss_seg);
2509 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2511 if (ret != X86EMUL_CONTINUE)
2512 /* FIXME: need to provide precise fault address */
2515 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2517 if (ret != X86EMUL_CONTINUE)
2518 /* FIXME: need to provide precise fault address */
2521 if (old_tss_sel != 0xffff) {
2522 tss_seg.prev_task_link = old_tss_sel;
2524 ret = ops->write_std(ctxt, new_tss_base,
2525 &tss_seg.prev_task_link,
2526 sizeof tss_seg.prev_task_link,
2528 if (ret != X86EMUL_CONTINUE)
2529 /* FIXME: need to provide precise fault address */
2533 return load_state_from_tss16(ctxt, &tss_seg);
2536 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2537 struct tss_segment_32 *tss)
2539 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2540 tss->eip = ctxt->_eip;
2541 tss->eflags = ctxt->eflags;
2542 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2543 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2544 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2545 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2546 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2547 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2548 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2549 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2551 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2552 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2553 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2554 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2555 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2556 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2557 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2560 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2561 struct tss_segment_32 *tss)
2565 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2566 return emulate_gp(ctxt, 0);
2567 ctxt->_eip = tss->eip;
2568 ctxt->eflags = tss->eflags | 2;
2570 /* General purpose registers */
2571 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2572 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2573 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2574 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2575 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2576 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2577 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2578 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2581 * SDM says that segment selectors are loaded before segment
2584 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2585 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2586 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2587 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2588 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2589 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2590 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2593 * If we're switching between Protected Mode and VM86, we need to make
2594 * sure to update the mode before loading the segment descriptors so
2595 * that the selectors are interpreted correctly.
2597 * Need to get rflags to the vcpu struct immediately because it
2598 * influences the CPL which is checked at least when loading the segment
2599 * descriptors and when pushing an error code to the new kernel stack.
2601 * TODO Introduce a separate ctxt->ops->set_cpl callback
2603 if (ctxt->eflags & X86_EFLAGS_VM)
2604 ctxt->mode = X86EMUL_MODE_VM86;
2606 ctxt->mode = X86EMUL_MODE_PROT32;
2608 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2611 * Now load segment descriptors. If fault happenes at this stage
2612 * it is handled in a context of new task
2614 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2615 if (ret != X86EMUL_CONTINUE)
2617 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2618 if (ret != X86EMUL_CONTINUE)
2620 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2621 if (ret != X86EMUL_CONTINUE)
2623 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2624 if (ret != X86EMUL_CONTINUE)
2626 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2627 if (ret != X86EMUL_CONTINUE)
2629 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2630 if (ret != X86EMUL_CONTINUE)
2632 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2633 if (ret != X86EMUL_CONTINUE)
2636 return X86EMUL_CONTINUE;
2639 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2640 u16 tss_selector, u16 old_tss_sel,
2641 ulong old_tss_base, struct desc_struct *new_desc)
2643 const struct x86_emulate_ops *ops = ctxt->ops;
2644 struct tss_segment_32 tss_seg;
2646 u32 new_tss_base = get_desc_base(new_desc);
2648 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2650 if (ret != X86EMUL_CONTINUE)
2651 /* FIXME: need to provide precise fault address */
2654 save_state_to_tss32(ctxt, &tss_seg);
2656 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2658 if (ret != X86EMUL_CONTINUE)
2659 /* FIXME: need to provide precise fault address */
2662 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2664 if (ret != X86EMUL_CONTINUE)
2665 /* FIXME: need to provide precise fault address */
2668 if (old_tss_sel != 0xffff) {
2669 tss_seg.prev_task_link = old_tss_sel;
2671 ret = ops->write_std(ctxt, new_tss_base,
2672 &tss_seg.prev_task_link,
2673 sizeof tss_seg.prev_task_link,
2675 if (ret != X86EMUL_CONTINUE)
2676 /* FIXME: need to provide precise fault address */
2680 return load_state_from_tss32(ctxt, &tss_seg);
2683 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2684 u16 tss_selector, int idt_index, int reason,
2685 bool has_error_code, u32 error_code)
2687 const struct x86_emulate_ops *ops = ctxt->ops;
2688 struct desc_struct curr_tss_desc, next_tss_desc;
2690 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2691 ulong old_tss_base =
2692 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2696 /* FIXME: old_tss_base == ~0 ? */
2698 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2699 if (ret != X86EMUL_CONTINUE)
2701 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2702 if (ret != X86EMUL_CONTINUE)
2705 /* FIXME: check that next_tss_desc is tss */
2708 * Check privileges. The three cases are task switch caused by...
2710 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2711 * 2. Exception/IRQ/iret: No check is performed
2712 * 3. jmp/call to TSS: Check against DPL of the TSS
2714 if (reason == TASK_SWITCH_GATE) {
2715 if (idt_index != -1) {
2716 /* Software interrupts */
2717 struct desc_struct task_gate_desc;
2720 ret = read_interrupt_descriptor(ctxt, idt_index,
2722 if (ret != X86EMUL_CONTINUE)
2725 dpl = task_gate_desc.dpl;
2726 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2727 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2729 } else if (reason != TASK_SWITCH_IRET) {
2730 int dpl = next_tss_desc.dpl;
2731 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2732 return emulate_gp(ctxt, tss_selector);
2736 desc_limit = desc_limit_scaled(&next_tss_desc);
2737 if (!next_tss_desc.p ||
2738 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2739 desc_limit < 0x2b)) {
2740 emulate_ts(ctxt, tss_selector & 0xfffc);
2741 return X86EMUL_PROPAGATE_FAULT;
2744 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2745 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2746 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2749 if (reason == TASK_SWITCH_IRET)
2750 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2752 /* set back link to prev task only if NT bit is set in eflags
2753 note that old_tss_sel is not used after this point */
2754 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2755 old_tss_sel = 0xffff;
2757 if (next_tss_desc.type & 8)
2758 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2759 old_tss_base, &next_tss_desc);
2761 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2762 old_tss_base, &next_tss_desc);
2763 if (ret != X86EMUL_CONTINUE)
2766 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2767 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2769 if (reason != TASK_SWITCH_IRET) {
2770 next_tss_desc.type |= (1 << 1); /* set busy flag */
2771 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2774 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2775 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2777 if (has_error_code) {
2778 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2779 ctxt->lock_prefix = 0;
2780 ctxt->src.val = (unsigned long) error_code;
2781 ret = em_push(ctxt);
2787 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2788 u16 tss_selector, int idt_index, int reason,
2789 bool has_error_code, u32 error_code)
2793 invalidate_registers(ctxt);
2794 ctxt->_eip = ctxt->eip;
2795 ctxt->dst.type = OP_NONE;
2797 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2798 has_error_code, error_code);
2800 if (rc == X86EMUL_CONTINUE) {
2801 ctxt->eip = ctxt->_eip;
2802 writeback_registers(ctxt);
2805 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2808 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2811 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2813 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2814 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2817 static int em_das(struct x86_emulate_ctxt *ctxt)
2820 bool af, cf, old_cf;
2822 cf = ctxt->eflags & X86_EFLAGS_CF;
2828 af = ctxt->eflags & X86_EFLAGS_AF;
2829 if ((al & 0x0f) > 9 || af) {
2831 cf = old_cf | (al >= 250);
2836 if (old_al > 0x99 || old_cf) {
2842 /* Set PF, ZF, SF */
2843 ctxt->src.type = OP_IMM;
2845 ctxt->src.bytes = 1;
2846 emulate_2op_SrcV(ctxt, "or");
2847 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2849 ctxt->eflags |= X86_EFLAGS_CF;
2851 ctxt->eflags |= X86_EFLAGS_AF;
2852 return X86EMUL_CONTINUE;
2855 static int em_call(struct x86_emulate_ctxt *ctxt)
2857 long rel = ctxt->src.val;
2859 ctxt->src.val = (unsigned long)ctxt->_eip;
2861 return em_push(ctxt);
2864 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2870 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2871 old_eip = ctxt->_eip;
2873 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2874 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2875 return X86EMUL_CONTINUE;
2878 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2880 ctxt->src.val = old_cs;
2882 if (rc != X86EMUL_CONTINUE)
2885 ctxt->src.val = old_eip;
2886 return em_push(ctxt);
2889 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2893 ctxt->dst.type = OP_REG;
2894 ctxt->dst.addr.reg = &ctxt->_eip;
2895 ctxt->dst.bytes = ctxt->op_bytes;
2896 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2897 if (rc != X86EMUL_CONTINUE)
2899 rsp_increment(ctxt, ctxt->src.val);
2900 return X86EMUL_CONTINUE;
2903 static int em_add(struct x86_emulate_ctxt *ctxt)
2905 emulate_2op_SrcV(ctxt, "add");
2906 return X86EMUL_CONTINUE;
2909 static int em_or(struct x86_emulate_ctxt *ctxt)
2911 emulate_2op_SrcV(ctxt, "or");
2912 return X86EMUL_CONTINUE;
2915 static int em_adc(struct x86_emulate_ctxt *ctxt)
2917 emulate_2op_SrcV(ctxt, "adc");
2918 return X86EMUL_CONTINUE;
2921 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2923 emulate_2op_SrcV(ctxt, "sbb");
2924 return X86EMUL_CONTINUE;
2927 static int em_and(struct x86_emulate_ctxt *ctxt)
2929 emulate_2op_SrcV(ctxt, "and");
2930 return X86EMUL_CONTINUE;
2933 static int em_sub(struct x86_emulate_ctxt *ctxt)
2935 emulate_2op_SrcV(ctxt, "sub");
2936 return X86EMUL_CONTINUE;
2939 static int em_xor(struct x86_emulate_ctxt *ctxt)
2941 emulate_2op_SrcV(ctxt, "xor");
2942 return X86EMUL_CONTINUE;
2945 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2947 emulate_2op_SrcV(ctxt, "cmp");
2948 /* Disable writeback. */
2949 ctxt->dst.type = OP_NONE;
2950 return X86EMUL_CONTINUE;
2953 static int em_test(struct x86_emulate_ctxt *ctxt)
2955 emulate_2op_SrcV(ctxt, "test");
2956 /* Disable writeback. */
2957 ctxt->dst.type = OP_NONE;
2958 return X86EMUL_CONTINUE;
2961 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2963 /* Write back the register source. */
2964 ctxt->src.val = ctxt->dst.val;
2965 write_register_operand(&ctxt->src);
2967 /* Write back the memory destination with implicit LOCK prefix. */
2968 ctxt->dst.val = ctxt->src.orig_val;
2969 ctxt->lock_prefix = 1;
2970 return X86EMUL_CONTINUE;
2973 static int em_imul(struct x86_emulate_ctxt *ctxt)
2975 emulate_2op_SrcV_nobyte(ctxt, "imul");
2976 return X86EMUL_CONTINUE;
2979 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2981 ctxt->dst.val = ctxt->src2.val;
2982 return em_imul(ctxt);
2985 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2987 ctxt->dst.type = OP_REG;
2988 ctxt->dst.bytes = ctxt->src.bytes;
2989 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2990 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2992 return X86EMUL_CONTINUE;
2995 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2999 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3000 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3001 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3002 return X86EMUL_CONTINUE;
3005 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3009 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3010 return emulate_gp(ctxt, 0);
3011 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3012 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3013 return X86EMUL_CONTINUE;
3016 static int em_mov(struct x86_emulate_ctxt *ctxt)
3018 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3019 return X86EMUL_CONTINUE;
3022 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3024 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3025 return emulate_gp(ctxt, 0);
3027 /* Disable writeback. */
3028 ctxt->dst.type = OP_NONE;
3029 return X86EMUL_CONTINUE;
3032 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3036 if (ctxt->mode == X86EMUL_MODE_PROT64)
3037 val = ctxt->src.val & ~0ULL;
3039 val = ctxt->src.val & ~0U;
3041 /* #UD condition is already handled. */
3042 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3043 return emulate_gp(ctxt, 0);
3045 /* Disable writeback. */
3046 ctxt->dst.type = OP_NONE;
3047 return X86EMUL_CONTINUE;
3050 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3054 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3055 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3056 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3057 return emulate_gp(ctxt, 0);
3059 return X86EMUL_CONTINUE;
3062 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3066 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3067 return emulate_gp(ctxt, 0);
3069 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3070 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3071 return X86EMUL_CONTINUE;
3074 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3076 if (ctxt->modrm_reg > VCPU_SREG_GS)
3077 return emulate_ud(ctxt);
3079 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3080 return X86EMUL_CONTINUE;
3083 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3085 u16 sel = ctxt->src.val;
3087 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3088 return emulate_ud(ctxt);
3090 if (ctxt->modrm_reg == VCPU_SREG_SS)
3091 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3093 /* Disable writeback. */
3094 ctxt->dst.type = OP_NONE;
3095 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3098 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3100 u16 sel = ctxt->src.val;
3102 /* Disable writeback. */
3103 ctxt->dst.type = OP_NONE;
3104 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3107 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3109 u16 sel = ctxt->src.val;
3111 /* Disable writeback. */
3112 ctxt->dst.type = OP_NONE;
3113 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3116 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3121 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3122 if (rc == X86EMUL_CONTINUE)
3123 ctxt->ops->invlpg(ctxt, linear);
3124 /* Disable writeback. */
3125 ctxt->dst.type = OP_NONE;
3126 return X86EMUL_CONTINUE;
3129 static int em_clts(struct x86_emulate_ctxt *ctxt)
3133 cr0 = ctxt->ops->get_cr(ctxt, 0);
3135 ctxt->ops->set_cr(ctxt, 0, cr0);
3136 return X86EMUL_CONTINUE;
3139 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3143 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3144 return X86EMUL_UNHANDLEABLE;
3146 rc = ctxt->ops->fix_hypercall(ctxt);
3147 if (rc != X86EMUL_CONTINUE)
3150 /* Let the processor re-execute the fixed hypercall */
3151 ctxt->_eip = ctxt->eip;
3152 /* Disable writeback. */
3153 ctxt->dst.type = OP_NONE;
3154 return X86EMUL_CONTINUE;
3157 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3158 void (*get)(struct x86_emulate_ctxt *ctxt,
3159 struct desc_ptr *ptr))
3161 struct desc_ptr desc_ptr;
3163 if (ctxt->mode == X86EMUL_MODE_PROT64)
3165 get(ctxt, &desc_ptr);
3166 if (ctxt->op_bytes == 2) {
3168 desc_ptr.address &= 0x00ffffff;
3170 /* Disable writeback. */
3171 ctxt->dst.type = OP_NONE;
3172 return segmented_write(ctxt, ctxt->dst.addr.mem,
3173 &desc_ptr, 2 + ctxt->op_bytes);
3176 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3178 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3181 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3183 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3186 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3188 struct desc_ptr desc_ptr;
3191 if (ctxt->mode == X86EMUL_MODE_PROT64)
3193 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3194 &desc_ptr.size, &desc_ptr.address,
3196 if (rc != X86EMUL_CONTINUE)
3198 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3199 /* Disable writeback. */
3200 ctxt->dst.type = OP_NONE;
3201 return X86EMUL_CONTINUE;
3204 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3208 rc = ctxt->ops->fix_hypercall(ctxt);
3210 /* Disable writeback. */
3211 ctxt->dst.type = OP_NONE;
3215 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3217 struct desc_ptr desc_ptr;
3220 if (ctxt->mode == X86EMUL_MODE_PROT64)
3222 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3223 &desc_ptr.size, &desc_ptr.address,
3225 if (rc != X86EMUL_CONTINUE)
3227 ctxt->ops->set_idt(ctxt, &desc_ptr);
3228 /* Disable writeback. */
3229 ctxt->dst.type = OP_NONE;
3230 return X86EMUL_CONTINUE;
3233 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3235 ctxt->dst.bytes = 2;
3236 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3237 return X86EMUL_CONTINUE;
3240 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3242 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3243 | (ctxt->src.val & 0x0f));
3244 ctxt->dst.type = OP_NONE;
3245 return X86EMUL_CONTINUE;
3248 static int em_loop(struct x86_emulate_ctxt *ctxt)
3250 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3251 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3252 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3253 jmp_rel(ctxt, ctxt->src.val);
3255 return X86EMUL_CONTINUE;
3258 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3260 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3261 jmp_rel(ctxt, ctxt->src.val);
3263 return X86EMUL_CONTINUE;
3266 static int em_in(struct x86_emulate_ctxt *ctxt)
3268 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3270 return X86EMUL_IO_NEEDED;
3272 return X86EMUL_CONTINUE;
3275 static int em_out(struct x86_emulate_ctxt *ctxt)
3277 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3279 /* Disable writeback. */
3280 ctxt->dst.type = OP_NONE;
3281 return X86EMUL_CONTINUE;
3284 static int em_cli(struct x86_emulate_ctxt *ctxt)
3286 if (emulator_bad_iopl(ctxt))
3287 return emulate_gp(ctxt, 0);
3289 ctxt->eflags &= ~X86_EFLAGS_IF;
3290 return X86EMUL_CONTINUE;
3293 static int em_sti(struct x86_emulate_ctxt *ctxt)
3295 if (emulator_bad_iopl(ctxt))
3296 return emulate_gp(ctxt, 0);
3298 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3299 ctxt->eflags |= X86_EFLAGS_IF;
3300 return X86EMUL_CONTINUE;
3303 static int em_bt(struct x86_emulate_ctxt *ctxt)
3305 /* Disable writeback. */
3306 ctxt->dst.type = OP_NONE;
3307 /* only subword offset */
3308 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3310 emulate_2op_SrcV_nobyte(ctxt, "bt");
3311 return X86EMUL_CONTINUE;
3314 static int em_bts(struct x86_emulate_ctxt *ctxt)
3316 emulate_2op_SrcV_nobyte(ctxt, "bts");
3317 return X86EMUL_CONTINUE;
3320 static int em_btr(struct x86_emulate_ctxt *ctxt)
3322 emulate_2op_SrcV_nobyte(ctxt, "btr");
3323 return X86EMUL_CONTINUE;
3326 static int em_btc(struct x86_emulate_ctxt *ctxt)
3328 emulate_2op_SrcV_nobyte(ctxt, "btc");
3329 return X86EMUL_CONTINUE;
3332 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3334 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3335 return X86EMUL_CONTINUE;
3338 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3340 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3341 return X86EMUL_CONTINUE;
3344 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3346 u32 eax, ebx, ecx, edx;
3348 eax = reg_read(ctxt, VCPU_REGS_RAX);
3349 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3350 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3351 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3352 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3353 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3354 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3355 return X86EMUL_CONTINUE;
3358 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3360 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3361 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3362 return X86EMUL_CONTINUE;
3365 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3367 switch (ctxt->op_bytes) {
3368 #ifdef CONFIG_X86_64
3370 asm("bswap %0" : "+r"(ctxt->dst.val));
3374 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3377 return X86EMUL_CONTINUE;
3380 static bool valid_cr(int nr)
3392 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3394 if (!valid_cr(ctxt->modrm_reg))
3395 return emulate_ud(ctxt);
3397 return X86EMUL_CONTINUE;
3400 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3402 u64 new_val = ctxt->src.val64;
3403 int cr = ctxt->modrm_reg;
3406 static u64 cr_reserved_bits[] = {
3407 0xffffffff00000000ULL,
3408 0, 0, 0, /* CR3 checked later */
3415 return emulate_ud(ctxt);
3417 if (new_val & cr_reserved_bits[cr])
3418 return emulate_gp(ctxt, 0);
3423 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3424 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3425 return emulate_gp(ctxt, 0);
3427 cr4 = ctxt->ops->get_cr(ctxt, 4);
3428 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3430 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3431 !(cr4 & X86_CR4_PAE))
3432 return emulate_gp(ctxt, 0);
3439 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3440 if (efer & EFER_LMA)
3441 rsvd = CR3_L_MODE_RESERVED_BITS;
3442 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3443 rsvd = CR3_PAE_RESERVED_BITS;
3444 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3445 rsvd = CR3_NONPAE_RESERVED_BITS;
3448 return emulate_gp(ctxt, 0);
3453 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3455 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3456 return emulate_gp(ctxt, 0);
3462 return X86EMUL_CONTINUE;
3465 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3469 ctxt->ops->get_dr(ctxt, 7, &dr7);
3471 /* Check if DR7.Global_Enable is set */
3472 return dr7 & (1 << 13);
3475 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3477 int dr = ctxt->modrm_reg;
3481 return emulate_ud(ctxt);
3483 cr4 = ctxt->ops->get_cr(ctxt, 4);
3484 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3485 return emulate_ud(ctxt);
3487 if (check_dr7_gd(ctxt))
3488 return emulate_db(ctxt);
3490 return X86EMUL_CONTINUE;
3493 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3495 u64 new_val = ctxt->src.val64;
3496 int dr = ctxt->modrm_reg;
3498 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3499 return emulate_gp(ctxt, 0);
3501 return check_dr_read(ctxt);
3504 static int check_svme(struct x86_emulate_ctxt *ctxt)
3508 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3510 if (!(efer & EFER_SVME))
3511 return emulate_ud(ctxt);
3513 return X86EMUL_CONTINUE;
3516 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3518 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3520 /* Valid physical address? */
3521 if (rax & 0xffff000000000000ULL)
3522 return emulate_gp(ctxt, 0);
3524 return check_svme(ctxt);
3527 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3529 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3531 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3532 return emulate_ud(ctxt);
3534 return X86EMUL_CONTINUE;
3537 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3539 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3540 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3542 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3544 return emulate_gp(ctxt, 0);
3546 return X86EMUL_CONTINUE;
3549 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3551 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3552 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3553 return emulate_gp(ctxt, 0);
3555 return X86EMUL_CONTINUE;
3558 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3560 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3561 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3562 return emulate_gp(ctxt, 0);
3564 return X86EMUL_CONTINUE;
3567 #define D(_y) { .flags = (_y) }
3568 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3569 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3570 .check_perm = (_p) }
3572 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3573 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3574 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3575 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3576 #define II(_f, _e, _i) \
3577 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3578 #define IIP(_f, _e, _i, _p) \
3579 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3580 .check_perm = (_p) }
3581 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3583 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3584 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3585 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3586 #define I2bvIP(_f, _e, _i, _p) \
3587 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3589 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3590 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3591 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3593 static const struct opcode group7_rm1[] = {
3594 DI(SrcNone | Priv, monitor),
3595 DI(SrcNone | Priv, mwait),
3599 static const struct opcode group7_rm3[] = {
3600 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3601 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3602 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3603 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3604 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3605 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3606 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3607 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3610 static const struct opcode group7_rm7[] = {
3612 DIP(SrcNone, rdtscp, check_rdtsc),
3616 static const struct opcode group1[] = {
3618 I(Lock | PageTable, em_or),
3621 I(Lock | PageTable, em_and),
3627 static const struct opcode group1A[] = {
3628 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3631 static const struct opcode group3[] = {
3632 I(DstMem | SrcImm, em_test),
3633 I(DstMem | SrcImm, em_test),
3634 I(DstMem | SrcNone | Lock, em_not),
3635 I(DstMem | SrcNone | Lock, em_neg),
3636 I(SrcMem, em_mul_ex),
3637 I(SrcMem, em_imul_ex),
3638 I(SrcMem, em_div_ex),
3639 I(SrcMem, em_idiv_ex),
3642 static const struct opcode group4[] = {
3643 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3644 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3648 static const struct opcode group5[] = {
3649 I(DstMem | SrcNone | Lock, em_grp45),
3650 I(DstMem | SrcNone | Lock, em_grp45),
3651 I(SrcMem | Stack, em_grp45),
3652 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3653 I(SrcMem | Stack, em_grp45),
3654 I(SrcMemFAddr | ImplicitOps, em_grp45),
3655 I(SrcMem | Stack, em_grp45), N,
3658 static const struct opcode group6[] = {
3661 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3662 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3666 static const struct group_dual group7 = { {
3667 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3668 II(Mov | DstMem | Priv, em_sidt, sidt),
3669 II(SrcMem | Priv, em_lgdt, lgdt),
3670 II(SrcMem | Priv, em_lidt, lidt),
3671 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3672 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3673 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3675 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3677 N, EXT(0, group7_rm3),
3678 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3679 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3683 static const struct opcode group8[] = {
3685 I(DstMem | SrcImmByte, em_bt),
3686 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3687 I(DstMem | SrcImmByte | Lock, em_btr),
3688 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3691 static const struct group_dual group9 = { {
3692 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3694 N, N, N, N, N, N, N, N,
3697 static const struct opcode group11[] = {
3698 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3702 static const struct gprefix pfx_0f_6f_0f_7f = {
3703 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3706 static const struct gprefix pfx_vmovntpx = {
3707 I(0, em_mov), N, N, N,
3710 static const struct opcode opcode_table[256] = {
3712 I6ALU(Lock, em_add),
3713 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3714 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3716 I6ALU(Lock | PageTable, em_or),
3717 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3720 I6ALU(Lock, em_adc),
3721 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3722 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3724 I6ALU(Lock, em_sbb),
3725 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3726 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3728 I6ALU(Lock | PageTable, em_and), N, N,
3730 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3732 I6ALU(Lock, em_xor), N, N,
3734 I6ALU(0, em_cmp), N, N,
3738 X8(I(SrcReg | Stack, em_push)),
3740 X8(I(DstReg | Stack, em_pop)),
3742 I(ImplicitOps | Stack | No64, em_pusha),
3743 I(ImplicitOps | Stack | No64, em_popa),
3744 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3747 I(SrcImm | Mov | Stack, em_push),
3748 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3749 I(SrcImmByte | Mov | Stack, em_push),
3750 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3751 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3752 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3756 G(ByteOp | DstMem | SrcImm, group1),
3757 G(DstMem | SrcImm, group1),
3758 G(ByteOp | DstMem | SrcImm | No64, group1),
3759 G(DstMem | SrcImmByte, group1),
3760 I2bv(DstMem | SrcReg | ModRM, em_test),
3761 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3763 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3764 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3765 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3766 D(ModRM | SrcMem | NoAccess | DstReg),
3767 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3770 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3772 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3773 I(SrcImmFAddr | No64, em_call_far), N,
3774 II(ImplicitOps | Stack, em_pushf, pushf),
3775 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3777 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3778 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3779 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3780 I2bv(SrcSI | DstDI | String, em_cmp),
3782 I2bv(DstAcc | SrcImm, em_test),
3783 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3784 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3785 I2bv(SrcAcc | DstDI | String, em_cmp),
3787 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3789 X8(I(DstReg | SrcImm | Mov, em_mov)),
3791 D2bv(DstMem | SrcImmByte | ModRM),
3792 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3793 I(ImplicitOps | Stack, em_ret),
3794 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3795 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3796 G(ByteOp, group11), G(0, group11),
3798 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3799 N, I(ImplicitOps | Stack, em_ret_far),
3800 D(ImplicitOps), DI(SrcImmByte, intn),
3801 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3803 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3806 N, N, N, N, N, N, N, N,
3808 X3(I(SrcImmByte, em_loop)),
3809 I(SrcImmByte, em_jcxz),
3810 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3811 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3813 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3814 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3815 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3816 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3818 N, DI(ImplicitOps, icebp), N, N,
3819 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3820 G(ByteOp, group3), G(0, group3),
3822 D(ImplicitOps), D(ImplicitOps),
3823 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3824 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3827 static const struct opcode twobyte_table[256] = {
3829 G(0, group6), GD(0, &group7), N, N,
3830 N, I(ImplicitOps | VendorSpecific, em_syscall),
3831 II(ImplicitOps | Priv, em_clts, clts), N,
3832 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3833 N, D(ImplicitOps | ModRM), N, N,
3835 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3837 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3838 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3839 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3840 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3842 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3845 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3846 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3847 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3848 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3849 I(ImplicitOps | VendorSpecific, em_sysenter),
3850 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3852 N, N, N, N, N, N, N, N,
3854 X16(D(DstReg | SrcMem | ModRM | Mov)),
3856 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3861 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3866 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3870 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3872 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3873 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3874 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3875 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3877 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3878 DI(ImplicitOps, rsm),
3879 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3880 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3881 D(DstMem | SrcReg | Src2CL | ModRM),
3882 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3884 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3885 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3886 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3887 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3888 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3889 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3893 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3894 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3895 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3897 D2bv(DstMem | SrcReg | ModRM | Lock),
3898 N, D(DstMem | SrcReg | ModRM | Mov),
3899 N, N, N, GD(0, &group9),
3901 X8(I(DstReg, em_bswap)),
3903 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3905 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3907 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3924 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3928 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3934 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3935 unsigned size, bool sign_extension)
3937 int rc = X86EMUL_CONTINUE;
3941 op->addr.mem.ea = ctxt->_eip;
3942 /* NB. Immediates are sign-extended as necessary. */
3943 switch (op->bytes) {
3945 op->val = insn_fetch(s8, ctxt);
3948 op->val = insn_fetch(s16, ctxt);
3951 op->val = insn_fetch(s32, ctxt);
3954 if (!sign_extension) {
3955 switch (op->bytes) {
3963 op->val &= 0xffffffff;
3971 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3974 int rc = X86EMUL_CONTINUE;
3978 decode_register_operand(ctxt, op);
3981 rc = decode_imm(ctxt, op, 1, false);
3984 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3988 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3989 fetch_bit_operand(ctxt);
3990 op->orig_val = op->val;
3993 ctxt->memop.bytes = 8;
3997 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3998 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3999 fetch_register_operand(op);
4000 op->orig_val = op->val;
4004 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4006 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4007 op->addr.mem.seg = VCPU_SREG_ES;
4014 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4015 fetch_register_operand(op);
4019 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4022 rc = decode_imm(ctxt, op, 1, true);
4029 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4032 ctxt->memop.bytes = 1;
4035 ctxt->memop.bytes = 2;
4038 ctxt->memop.bytes = 4;
4041 rc = decode_imm(ctxt, op, 2, false);
4044 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4048 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4050 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4051 op->addr.mem.seg = seg_override(ctxt);
4057 op->addr.mem.ea = ctxt->_eip;
4058 op->bytes = ctxt->op_bytes + 2;
4059 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4062 ctxt->memop.bytes = ctxt->op_bytes + 2;
4065 op->val = VCPU_SREG_ES;
4068 op->val = VCPU_SREG_CS;
4071 op->val = VCPU_SREG_SS;
4074 op->val = VCPU_SREG_DS;
4077 op->val = VCPU_SREG_FS;
4080 op->val = VCPU_SREG_GS;
4083 /* Special instructions do their own operand decoding. */
4085 op->type = OP_NONE; /* Disable writeback. */
4093 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4095 int rc = X86EMUL_CONTINUE;
4096 int mode = ctxt->mode;
4097 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4098 bool op_prefix = false;
4099 struct opcode opcode;
4101 ctxt->memop.type = OP_NONE;
4102 ctxt->memopp = NULL;
4103 ctxt->_eip = ctxt->eip;
4104 ctxt->fetch.start = ctxt->_eip;
4105 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4107 memcpy(ctxt->fetch.data, insn, insn_len);
4110 case X86EMUL_MODE_REAL:
4111 case X86EMUL_MODE_VM86:
4112 case X86EMUL_MODE_PROT16:
4113 def_op_bytes = def_ad_bytes = 2;
4115 case X86EMUL_MODE_PROT32:
4116 def_op_bytes = def_ad_bytes = 4;
4118 #ifdef CONFIG_X86_64
4119 case X86EMUL_MODE_PROT64:
4125 return EMULATION_FAILED;
4128 ctxt->op_bytes = def_op_bytes;
4129 ctxt->ad_bytes = def_ad_bytes;
4131 /* Legacy prefixes. */
4133 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4134 case 0x66: /* operand-size override */
4136 /* switch between 2/4 bytes */
4137 ctxt->op_bytes = def_op_bytes ^ 6;
4139 case 0x67: /* address-size override */
4140 if (mode == X86EMUL_MODE_PROT64)
4141 /* switch between 4/8 bytes */
4142 ctxt->ad_bytes = def_ad_bytes ^ 12;
4144 /* switch between 2/4 bytes */
4145 ctxt->ad_bytes = def_ad_bytes ^ 6;
4147 case 0x26: /* ES override */
4148 case 0x2e: /* CS override */
4149 case 0x36: /* SS override */
4150 case 0x3e: /* DS override */
4151 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4153 case 0x64: /* FS override */
4154 case 0x65: /* GS override */
4155 set_seg_override(ctxt, ctxt->b & 7);
4157 case 0x40 ... 0x4f: /* REX */
4158 if (mode != X86EMUL_MODE_PROT64)
4160 ctxt->rex_prefix = ctxt->b;
4162 case 0xf0: /* LOCK */
4163 ctxt->lock_prefix = 1;
4165 case 0xf2: /* REPNE/REPNZ */
4166 case 0xf3: /* REP/REPE/REPZ */
4167 ctxt->rep_prefix = ctxt->b;
4173 /* Any legacy prefix after a REX prefix nullifies its effect. */
4175 ctxt->rex_prefix = 0;
4181 if (ctxt->rex_prefix & 8)
4182 ctxt->op_bytes = 8; /* REX.W */
4184 /* Opcode byte(s). */
4185 opcode = opcode_table[ctxt->b];
4186 /* Two-byte opcode? */
4187 if (ctxt->b == 0x0f) {
4189 ctxt->b = insn_fetch(u8, ctxt);
4190 opcode = twobyte_table[ctxt->b];
4192 ctxt->d = opcode.flags;
4194 if (ctxt->d & ModRM)
4195 ctxt->modrm = insn_fetch(u8, ctxt);
4197 while (ctxt->d & GroupMask) {
4198 switch (ctxt->d & GroupMask) {
4200 goffset = (ctxt->modrm >> 3) & 7;
4201 opcode = opcode.u.group[goffset];
4204 goffset = (ctxt->modrm >> 3) & 7;
4205 if ((ctxt->modrm >> 6) == 3)
4206 opcode = opcode.u.gdual->mod3[goffset];
4208 opcode = opcode.u.gdual->mod012[goffset];
4211 goffset = ctxt->modrm & 7;
4212 opcode = opcode.u.group[goffset];
4215 if (ctxt->rep_prefix && op_prefix)
4216 return EMULATION_FAILED;
4217 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4218 switch (simd_prefix) {
4219 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4220 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4221 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4222 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4226 return EMULATION_FAILED;
4229 ctxt->d &= ~(u64)GroupMask;
4230 ctxt->d |= opcode.flags;
4233 ctxt->execute = opcode.u.execute;
4234 ctxt->check_perm = opcode.check_perm;
4235 ctxt->intercept = opcode.intercept;
4238 if (ctxt->d == 0 || (ctxt->d & Undefined))
4239 return EMULATION_FAILED;
4241 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4242 return EMULATION_FAILED;
4244 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4247 if (ctxt->d & Op3264) {
4248 if (mode == X86EMUL_MODE_PROT64)
4255 ctxt->op_bytes = 16;
4256 else if (ctxt->d & Mmx)
4259 /* ModRM and SIB bytes. */
4260 if (ctxt->d & ModRM) {
4261 rc = decode_modrm(ctxt, &ctxt->memop);
4262 if (!ctxt->has_seg_override)
4263 set_seg_override(ctxt, ctxt->modrm_seg);
4264 } else if (ctxt->d & MemAbs)
4265 rc = decode_abs(ctxt, &ctxt->memop);
4266 if (rc != X86EMUL_CONTINUE)
4269 if (!ctxt->has_seg_override)
4270 set_seg_override(ctxt, VCPU_SREG_DS);
4272 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4274 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4275 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4278 * Decode and fetch the source operand: register, memory
4281 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4282 if (rc != X86EMUL_CONTINUE)
4286 * Decode and fetch the second source operand: register, memory
4289 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4290 if (rc != X86EMUL_CONTINUE)
4293 /* Decode and fetch the destination operand: register or memory. */
4294 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4297 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4298 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4300 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4303 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4305 return ctxt->d & PageTable;
4308 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4310 /* The second termination condition only applies for REPE
4311 * and REPNE. Test if the repeat string operation prefix is
4312 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4313 * corresponding termination condition according to:
4314 * - if REPE/REPZ and ZF = 0 then done
4315 * - if REPNE/REPNZ and ZF = 1 then done
4317 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4318 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4319 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4320 ((ctxt->eflags & EFLG_ZF) == 0))
4321 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4322 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4328 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4332 ctxt->ops->get_fpu(ctxt);
4333 asm volatile("1: fwait \n\t"
4335 ".pushsection .fixup,\"ax\" \n\t"
4337 "movb $1, %[fault] \n\t"
4340 _ASM_EXTABLE(1b, 3b)
4341 : [fault]"+qm"(fault));
4342 ctxt->ops->put_fpu(ctxt);
4344 if (unlikely(fault))
4345 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4347 return X86EMUL_CONTINUE;
4350 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4353 if (op->type == OP_MM)
4354 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4358 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4360 const struct x86_emulate_ops *ops = ctxt->ops;
4361 int rc = X86EMUL_CONTINUE;
4362 int saved_dst_type = ctxt->dst.type;
4364 ctxt->mem_read.pos = 0;
4366 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4367 rc = emulate_ud(ctxt);
4371 /* LOCK prefix is allowed only with some instructions */
4372 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4373 rc = emulate_ud(ctxt);
4377 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4378 rc = emulate_ud(ctxt);
4382 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4383 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4384 rc = emulate_ud(ctxt);
4388 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4389 rc = emulate_nm(ctxt);
4393 if (ctxt->d & Mmx) {
4394 rc = flush_pending_x87_faults(ctxt);
4395 if (rc != X86EMUL_CONTINUE)
4398 * Now that we know the fpu is exception safe, we can fetch
4401 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4402 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4403 if (!(ctxt->d & Mov))
4404 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4407 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4408 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4409 X86_ICPT_PRE_EXCEPT);
4410 if (rc != X86EMUL_CONTINUE)
4414 /* Privileged instruction can be executed only in CPL=0 */
4415 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4416 rc = emulate_gp(ctxt, 0);
4420 /* Instruction can only be executed in protected mode */
4421 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4422 rc = emulate_ud(ctxt);
4426 /* Do instruction specific permission checks */
4427 if (ctxt->check_perm) {
4428 rc = ctxt->check_perm(ctxt);
4429 if (rc != X86EMUL_CONTINUE)
4433 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4434 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4435 X86_ICPT_POST_EXCEPT);
4436 if (rc != X86EMUL_CONTINUE)
4440 if (ctxt->rep_prefix && (ctxt->d & String)) {
4441 /* All REP prefixes have the same first termination condition */
4442 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4443 ctxt->eip = ctxt->_eip;
4448 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4449 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4450 ctxt->src.valptr, ctxt->src.bytes);
4451 if (rc != X86EMUL_CONTINUE)
4453 ctxt->src.orig_val64 = ctxt->src.val64;
4456 if (ctxt->src2.type == OP_MEM) {
4457 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4458 &ctxt->src2.val, ctxt->src2.bytes);
4459 if (rc != X86EMUL_CONTINUE)
4463 if ((ctxt->d & DstMask) == ImplicitOps)
4467 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4468 /* optimisation - avoid slow emulated read if Mov */
4469 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4470 &ctxt->dst.val, ctxt->dst.bytes);
4471 if (rc != X86EMUL_CONTINUE)
4474 ctxt->dst.orig_val = ctxt->dst.val;
4478 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4479 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4480 X86_ICPT_POST_MEMACCESS);
4481 if (rc != X86EMUL_CONTINUE)
4485 if (ctxt->execute) {
4486 rc = ctxt->execute(ctxt);
4487 if (rc != X86EMUL_CONTINUE)
4496 case 0x40 ... 0x47: /* inc r16/r32 */
4497 emulate_1op(ctxt, "inc");
4499 case 0x48 ... 0x4f: /* dec r16/r32 */
4500 emulate_1op(ctxt, "dec");
4502 case 0x63: /* movsxd */
4503 if (ctxt->mode != X86EMUL_MODE_PROT64)
4504 goto cannot_emulate;
4505 ctxt->dst.val = (s32) ctxt->src.val;
4507 case 0x70 ... 0x7f: /* jcc (short) */
4508 if (test_cc(ctxt->b, ctxt->eflags))
4509 jmp_rel(ctxt, ctxt->src.val);
4511 case 0x8d: /* lea r16/r32, m */
4512 ctxt->dst.val = ctxt->src.addr.mem.ea;
4514 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4515 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4519 case 0x98: /* cbw/cwde/cdqe */
4520 switch (ctxt->op_bytes) {
4521 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4522 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4523 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4529 case 0xcc: /* int3 */
4530 rc = emulate_int(ctxt, 3);
4532 case 0xcd: /* int n */
4533 rc = emulate_int(ctxt, ctxt->src.val);
4535 case 0xce: /* into */
4536 if (ctxt->eflags & EFLG_OF)
4537 rc = emulate_int(ctxt, 4);
4539 case 0xd0 ... 0xd1: /* Grp2 */
4542 case 0xd2 ... 0xd3: /* Grp2 */
4543 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4546 case 0xe9: /* jmp rel */
4547 case 0xeb: /* jmp rel short */
4548 jmp_rel(ctxt, ctxt->src.val);
4549 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4551 case 0xf4: /* hlt */
4552 ctxt->ops->halt(ctxt);
4554 case 0xf5: /* cmc */
4555 /* complement carry flag from eflags reg */
4556 ctxt->eflags ^= EFLG_CF;
4558 case 0xf8: /* clc */
4559 ctxt->eflags &= ~EFLG_CF;
4561 case 0xf9: /* stc */
4562 ctxt->eflags |= EFLG_CF;
4564 case 0xfc: /* cld */
4565 ctxt->eflags &= ~EFLG_DF;
4567 case 0xfd: /* std */
4568 ctxt->eflags |= EFLG_DF;
4571 goto cannot_emulate;
4574 if (rc != X86EMUL_CONTINUE)
4578 rc = writeback(ctxt);
4579 if (rc != X86EMUL_CONTINUE)
4583 * restore dst type in case the decoding will be reused
4584 * (happens for string instruction )
4586 ctxt->dst.type = saved_dst_type;
4588 if ((ctxt->d & SrcMask) == SrcSI)
4589 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4591 if ((ctxt->d & DstMask) == DstDI)
4592 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4594 if (ctxt->rep_prefix && (ctxt->d & String)) {
4596 struct read_cache *r = &ctxt->io_read;
4597 if ((ctxt->d & SrcMask) == SrcSI)
4598 count = ctxt->src.count;
4600 count = ctxt->dst.count;
4601 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4604 if (!string_insn_completed(ctxt)) {
4606 * Re-enter guest when pio read ahead buffer is empty
4607 * or, if it is not used, after each 1024 iteration.
4609 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4610 (r->end == 0 || r->end != r->pos)) {
4612 * Reset read cache. Usually happens before
4613 * decode, but since instruction is restarted
4614 * we have to do it here.
4616 ctxt->mem_read.end = 0;
4617 writeback_registers(ctxt);
4618 return EMULATION_RESTART;
4620 goto done; /* skip rip writeback */
4624 ctxt->eip = ctxt->_eip;
4627 if (rc == X86EMUL_PROPAGATE_FAULT)
4628 ctxt->have_exception = true;
4629 if (rc == X86EMUL_INTERCEPTED)
4630 return EMULATION_INTERCEPTED;
4632 if (rc == X86EMUL_CONTINUE)
4633 writeback_registers(ctxt);
4635 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4639 case 0x09: /* wbinvd */
4640 (ctxt->ops->wbinvd)(ctxt);
4642 case 0x08: /* invd */
4643 case 0x0d: /* GrpP (prefetch) */
4644 case 0x18: /* Grp16 (prefetch/nop) */
4646 case 0x20: /* mov cr, reg */
4647 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4649 case 0x21: /* mov from dr to reg */
4650 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4652 case 0x40 ... 0x4f: /* cmov */
4653 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4654 if (!test_cc(ctxt->b, ctxt->eflags))
4655 ctxt->dst.type = OP_NONE; /* no writeback */
4657 case 0x80 ... 0x8f: /* jnz rel, etc*/
4658 if (test_cc(ctxt->b, ctxt->eflags))
4659 jmp_rel(ctxt, ctxt->src.val);
4661 case 0x90 ... 0x9f: /* setcc r/m8 */
4662 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4664 case 0xa4: /* shld imm8, r, r/m */
4665 case 0xa5: /* shld cl, r, r/m */
4666 emulate_2op_cl(ctxt, "shld");
4668 case 0xac: /* shrd imm8, r, r/m */
4669 case 0xad: /* shrd cl, r, r/m */
4670 emulate_2op_cl(ctxt, "shrd");
4672 case 0xae: /* clflush */
4674 case 0xb6 ... 0xb7: /* movzx */
4675 ctxt->dst.bytes = ctxt->op_bytes;
4676 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4677 : (u16) ctxt->src.val;
4679 case 0xbe ... 0xbf: /* movsx */
4680 ctxt->dst.bytes = ctxt->op_bytes;
4681 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4682 (s16) ctxt->src.val;
4684 case 0xc0 ... 0xc1: /* xadd */
4685 emulate_2op_SrcV(ctxt, "add");
4686 /* Write back the register source. */
4687 ctxt->src.val = ctxt->dst.orig_val;
4688 write_register_operand(&ctxt->src);
4690 case 0xc3: /* movnti */
4691 ctxt->dst.bytes = ctxt->op_bytes;
4692 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4693 (u64) ctxt->src.val;
4696 goto cannot_emulate;
4699 if (rc != X86EMUL_CONTINUE)
4705 return EMULATION_FAILED;
4708 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4710 invalidate_registers(ctxt);
4713 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4715 writeback_registers(ctxt);