1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
28 #include <asm/debugreg.h>
37 #define OpImplicit 1ull /* No generic decode */
38 #define OpReg 2ull /* Register */
39 #define OpMem 3ull /* Memory */
40 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
41 #define OpDI 5ull /* ES:DI/EDI/RDI */
42 #define OpMem64 6ull /* Memory, 64-bit */
43 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
44 #define OpDX 8ull /* DX register */
45 #define OpCL 9ull /* CL register (for shifts) */
46 #define OpImmByte 10ull /* 8-bit sign extended immediate */
47 #define OpOne 11ull /* Implied 1 */
48 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
49 #define OpMem16 13ull /* Memory operand (16-bit). */
50 #define OpMem32 14ull /* Memory operand (32-bit). */
51 #define OpImmU 15ull /* Immediate operand, zero extended */
52 #define OpSI 16ull /* SI/ESI/RSI */
53 #define OpImmFAddr 17ull /* Immediate far address */
54 #define OpMemFAddr 18ull /* Far address in memory */
55 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
56 #define OpES 20ull /* ES */
57 #define OpCS 21ull /* CS */
58 #define OpSS 22ull /* SS */
59 #define OpDS 23ull /* DS */
60 #define OpFS 24ull /* FS */
61 #define OpGS 25ull /* GS */
62 #define OpMem8 26ull /* 8-bit zero extended memory operand */
63 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
65 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
66 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
68 #define OpBits 5 /* Width of operand field */
69 #define OpMask ((1ull << OpBits) - 1)
72 * Opcode effective-address decode tables.
73 * Note that we only emulate instructions that have at least one memory
74 * operand (excluding implicit stack references). We assume that stack
75 * references and instruction fetches will never occur in special memory
76 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
80 /* Operand sizes: 8-bit operands or specified/overridden size. */
81 #define ByteOp (1<<0) /* 8-bit operands. */
82 /* Destination operand type. */
84 #define ImplicitOps (OpImplicit << DstShift)
85 #define DstReg (OpReg << DstShift)
86 #define DstMem (OpMem << DstShift)
87 #define DstAcc (OpAcc << DstShift)
88 #define DstDI (OpDI << DstShift)
89 #define DstMem64 (OpMem64 << DstShift)
90 #define DstMem16 (OpMem16 << DstShift)
91 #define DstImmUByte (OpImmUByte << DstShift)
92 #define DstDX (OpDX << DstShift)
93 #define DstAccLo (OpAccLo << DstShift)
94 #define DstMask (OpMask << DstShift)
95 /* Source operand type. */
97 #define SrcNone (OpNone << SrcShift)
98 #define SrcReg (OpReg << SrcShift)
99 #define SrcMem (OpMem << SrcShift)
100 #define SrcMem16 (OpMem16 << SrcShift)
101 #define SrcMem32 (OpMem32 << SrcShift)
102 #define SrcImm (OpImm << SrcShift)
103 #define SrcImmByte (OpImmByte << SrcShift)
104 #define SrcOne (OpOne << SrcShift)
105 #define SrcImmUByte (OpImmUByte << SrcShift)
106 #define SrcImmU (OpImmU << SrcShift)
107 #define SrcSI (OpSI << SrcShift)
108 #define SrcXLat (OpXLat << SrcShift)
109 #define SrcImmFAddr (OpImmFAddr << SrcShift)
110 #define SrcMemFAddr (OpMemFAddr << SrcShift)
111 #define SrcAcc (OpAcc << SrcShift)
112 #define SrcImmU16 (OpImmU16 << SrcShift)
113 #define SrcImm64 (OpImm64 << SrcShift)
114 #define SrcDX (OpDX << SrcShift)
115 #define SrcMem8 (OpMem8 << SrcShift)
116 #define SrcAccHi (OpAccHi << SrcShift)
117 #define SrcMask (OpMask << SrcShift)
118 #define BitOp (1<<11)
119 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
120 #define String (1<<13) /* String instruction (rep capable) */
121 #define Stack (1<<14) /* Stack instruction (push/pop) */
122 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
123 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
124 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
125 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
126 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
127 #define Escape (5<<15) /* Escape to coprocessor instruction */
128 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
129 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
130 #define Sse (1<<18) /* SSE Vector instruction */
131 /* Generic ModRM decode. */
132 #define ModRM (1<<19)
133 /* Destination is only written; never read. */
136 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
137 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
138 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
139 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
140 #define Undefined (1<<25) /* No Such Instruction */
141 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
142 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
144 #define PageTable (1 << 29) /* instruction used to write page table */
145 #define NotImpl (1 << 30) /* instruction is not implemented */
146 /* Source 2 operand type */
147 #define Src2Shift (31)
148 #define Src2None (OpNone << Src2Shift)
149 #define Src2Mem (OpMem << Src2Shift)
150 #define Src2CL (OpCL << Src2Shift)
151 #define Src2ImmByte (OpImmByte << Src2Shift)
152 #define Src2One (OpOne << Src2Shift)
153 #define Src2Imm (OpImm << Src2Shift)
154 #define Src2ES (OpES << Src2Shift)
155 #define Src2CS (OpCS << Src2Shift)
156 #define Src2SS (OpSS << Src2Shift)
157 #define Src2DS (OpDS << Src2Shift)
158 #define Src2FS (OpFS << Src2Shift)
159 #define Src2GS (OpGS << Src2Shift)
160 #define Src2Mask (OpMask << Src2Shift)
161 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
162 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
163 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
164 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
165 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
166 #define NoWrite ((u64)1 << 45) /* No writeback */
167 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
168 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
169 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
170 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
171 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
172 #define NearBranch ((u64)1 << 52) /* Near branches */
173 #define No16 ((u64)1 << 53) /* No 16 bit operand */
174 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
176 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
178 #define X2(x...) x, x
179 #define X3(x...) X2(x), x
180 #define X4(x...) X2(x), X2(x)
181 #define X5(x...) X4(x), x
182 #define X6(x...) X4(x), X2(x)
183 #define X7(x...) X4(x), X3(x)
184 #define X8(x...) X4(x), X4(x)
185 #define X16(x...) X8(x), X8(x)
187 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
188 #define FASTOP_SIZE 8
191 * fastop functions have a special calling convention:
196 * flags: rflags (in/out)
197 * ex: rsi (in:fastop pointer, out:zero if exception)
199 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
200 * different operand sizes can be reached by calculation, rather than a jump
201 * table (which would be bigger than the code).
203 * fastop functions are declared as taking a never-defined fastop parameter,
204 * so they can't be called from C directly.
213 int (*execute)(struct x86_emulate_ctxt *ctxt);
214 const struct opcode *group;
215 const struct group_dual *gdual;
216 const struct gprefix *gprefix;
217 const struct escape *esc;
218 const struct instr_dual *idual;
219 const struct mode_dual *mdual;
220 void (*fastop)(struct fastop *fake);
222 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
226 struct opcode mod012[8];
227 struct opcode mod3[8];
231 struct opcode pfx_no;
232 struct opcode pfx_66;
233 struct opcode pfx_f2;
234 struct opcode pfx_f3;
239 struct opcode high[64];
243 struct opcode mod012;
248 struct opcode mode32;
249 struct opcode mode64;
252 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 enum x86_transfer_type {
256 X86_TRANSFER_CALL_JMP,
258 X86_TRANSFER_TASK_SWITCH,
261 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
263 if (!(ctxt->regs_valid & (1 << nr))) {
264 ctxt->regs_valid |= 1 << nr;
265 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
267 return ctxt->_regs[nr];
270 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
272 ctxt->regs_valid |= 1 << nr;
273 ctxt->regs_dirty |= 1 << nr;
274 return &ctxt->_regs[nr];
277 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
280 return reg_write(ctxt, nr);
283 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
287 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
288 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
291 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
293 ctxt->regs_dirty = 0;
294 ctxt->regs_valid = 0;
298 * These EFLAGS bits are restored from saved value during emulation, and
299 * any changes are written back to the saved value after emulation.
301 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
302 X86_EFLAGS_PF|X86_EFLAGS_CF)
310 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
312 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
313 #define FOP_RET "ret \n\t"
315 #define FOP_START(op) \
316 extern void em_##op(struct fastop *fake); \
317 asm(".pushsection .text, \"ax\" \n\t" \
318 ".global em_" #op " \n\t" \
325 #define FOPNOP() FOP_ALIGN FOP_RET
327 #define FOP1E(op, dst) \
328 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
330 #define FOP1EEX(op, dst) \
331 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
333 #define FASTOP1(op) \
338 ON64(FOP1E(op##q, rax)) \
341 /* 1-operand, using src2 (for MUL/DIV r/m) */
342 #define FASTOP1SRC2(op, name) \
347 ON64(FOP1E(op, rcx)) \
350 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
351 #define FASTOP1SRC2EX(op, name) \
356 ON64(FOP1EEX(op, rcx)) \
359 #define FOP2E(op, dst, src) \
360 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
362 #define FASTOP2(op) \
364 FOP2E(op##b, al, dl) \
365 FOP2E(op##w, ax, dx) \
366 FOP2E(op##l, eax, edx) \
367 ON64(FOP2E(op##q, rax, rdx)) \
370 /* 2 operand, word only */
371 #define FASTOP2W(op) \
374 FOP2E(op##w, ax, dx) \
375 FOP2E(op##l, eax, edx) \
376 ON64(FOP2E(op##q, rax, rdx)) \
379 /* 2 operand, src is CL */
380 #define FASTOP2CL(op) \
382 FOP2E(op##b, al, cl) \
383 FOP2E(op##w, ax, cl) \
384 FOP2E(op##l, eax, cl) \
385 ON64(FOP2E(op##q, rax, cl)) \
388 /* 2 operand, src and dest are reversed */
389 #define FASTOP2R(op, name) \
391 FOP2E(op##b, dl, al) \
392 FOP2E(op##w, dx, ax) \
393 FOP2E(op##l, edx, eax) \
394 ON64(FOP2E(op##q, rdx, rax)) \
397 #define FOP3E(op, dst, src, src2) \
398 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
400 /* 3-operand, word-only, src2=cl */
401 #define FASTOP3WCL(op) \
404 FOP3E(op##w, ax, dx, cl) \
405 FOP3E(op##l, eax, edx, cl) \
406 ON64(FOP3E(op##q, rax, rdx, cl)) \
409 /* Special case for SETcc - 1 instruction per cc */
410 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
412 asm(".global kvm_fastop_exception \n"
413 "kvm_fastop_exception: xor %esi, %esi; ret");
434 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
437 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
438 enum x86_intercept intercept,
439 enum x86_intercept_stage stage)
441 struct x86_instruction_info info = {
442 .intercept = intercept,
443 .rep_prefix = ctxt->rep_prefix,
444 .modrm_mod = ctxt->modrm_mod,
445 .modrm_reg = ctxt->modrm_reg,
446 .modrm_rm = ctxt->modrm_rm,
447 .src_val = ctxt->src.val64,
448 .dst_val = ctxt->dst.val64,
449 .src_bytes = ctxt->src.bytes,
450 .dst_bytes = ctxt->dst.bytes,
451 .ad_bytes = ctxt->ad_bytes,
452 .next_rip = ctxt->eip,
455 return ctxt->ops->intercept(ctxt, &info, stage);
458 static void assign_masked(ulong *dest, ulong src, ulong mask)
460 *dest = (*dest & ~mask) | (src & mask);
463 static void assign_register(unsigned long *reg, u64 val, int bytes)
465 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
468 *(u8 *)reg = (u8)val;
471 *(u16 *)reg = (u16)val;
475 break; /* 64b: zero-extend */
482 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
484 return (1UL << (ctxt->ad_bytes << 3)) - 1;
487 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
490 struct desc_struct ss;
492 if (ctxt->mode == X86EMUL_MODE_PROT64)
494 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
495 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
498 static int stack_size(struct x86_emulate_ctxt *ctxt)
500 return (__fls(stack_mask(ctxt)) + 1) >> 3;
503 /* Access/update address held in a register, based on addressing mode. */
504 static inline unsigned long
505 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
507 if (ctxt->ad_bytes == sizeof(unsigned long))
510 return reg & ad_mask(ctxt);
513 static inline unsigned long
514 register_address(struct x86_emulate_ctxt *ctxt, int reg)
516 return address_mask(ctxt, reg_read(ctxt, reg));
519 static void masked_increment(ulong *reg, ulong mask, int inc)
521 assign_masked(reg, *reg + inc, mask);
525 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
527 ulong *preg = reg_rmw(ctxt, reg);
529 assign_register(preg, *preg + inc, ctxt->ad_bytes);
532 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
534 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
537 static u32 desc_limit_scaled(struct desc_struct *desc)
539 u32 limit = get_desc_limit(desc);
541 return desc->g ? (limit << 12) | 0xfff : limit;
544 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
546 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
549 return ctxt->ops->get_cached_segment_base(ctxt, seg);
552 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
553 u32 error, bool valid)
556 ctxt->exception.vector = vec;
557 ctxt->exception.error_code = error;
558 ctxt->exception.error_code_valid = valid;
559 return X86EMUL_PROPAGATE_FAULT;
562 static int emulate_db(struct x86_emulate_ctxt *ctxt)
564 return emulate_exception(ctxt, DB_VECTOR, 0, false);
567 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
569 return emulate_exception(ctxt, GP_VECTOR, err, true);
572 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
574 return emulate_exception(ctxt, SS_VECTOR, err, true);
577 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
579 return emulate_exception(ctxt, UD_VECTOR, 0, false);
582 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
584 return emulate_exception(ctxt, TS_VECTOR, err, true);
587 static int emulate_de(struct x86_emulate_ctxt *ctxt)
589 return emulate_exception(ctxt, DE_VECTOR, 0, false);
592 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
594 return emulate_exception(ctxt, NM_VECTOR, 0, false);
597 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
600 struct desc_struct desc;
602 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
606 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
611 struct desc_struct desc;
613 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
614 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
618 * x86 defines three classes of vector instructions: explicitly
619 * aligned, explicitly unaligned, and the rest, which change behaviour
620 * depending on whether they're AVX encoded or not.
622 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
623 * subject to the same check.
625 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
627 if (likely(size < 16))
630 if (ctxt->d & Aligned)
632 else if (ctxt->d & Unaligned)
634 else if (ctxt->d & Avx)
640 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
641 struct segmented_address addr,
642 unsigned *max_size, unsigned size,
643 bool write, bool fetch,
644 enum x86emul_mode mode, ulong *linear)
646 struct desc_struct desc;
652 la = seg_base(ctxt, addr.seg) + addr.ea;
655 case X86EMUL_MODE_PROT64:
656 if (is_noncanonical_address(la))
659 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
660 if (size > *max_size)
664 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
668 /* code segment in protected mode or read-only data segment */
669 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
670 || !(desc.type & 2)) && write)
672 /* unreadable code segment */
673 if (!fetch && (desc.type & 8) && !(desc.type & 2))
675 lim = desc_limit_scaled(&desc);
676 if (!(desc.type & 8) && (desc.type & 4)) {
677 /* expand-down segment */
680 lim = desc.d ? 0xffffffff : 0xffff;
684 if (lim == 0xffffffff)
687 *max_size = (u64)lim + 1 - addr.ea;
688 if (size > *max_size)
694 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
695 return emulate_gp(ctxt, 0);
697 return X86EMUL_CONTINUE;
699 if (addr.seg == VCPU_SREG_SS)
700 return emulate_ss(ctxt, 0);
702 return emulate_gp(ctxt, 0);
705 static int linearize(struct x86_emulate_ctxt *ctxt,
706 struct segmented_address addr,
707 unsigned size, bool write,
711 return __linearize(ctxt, addr, &max_size, size, write, false,
715 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
716 enum x86emul_mode mode)
721 struct segmented_address addr = { .seg = VCPU_SREG_CS,
724 if (ctxt->op_bytes != sizeof(unsigned long))
725 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
726 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
727 if (rc == X86EMUL_CONTINUE)
728 ctxt->_eip = addr.ea;
732 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
734 return assign_eip(ctxt, dst, ctxt->mode);
737 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
738 const struct desc_struct *cs_desc)
740 enum x86emul_mode mode = ctxt->mode;
744 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
748 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
750 mode = X86EMUL_MODE_PROT64;
752 mode = X86EMUL_MODE_PROT32; /* temporary value */
755 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
756 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
757 rc = assign_eip(ctxt, dst, mode);
758 if (rc == X86EMUL_CONTINUE)
763 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
765 return assign_eip_near(ctxt, ctxt->_eip + rel);
768 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
769 struct segmented_address addr,
776 rc = linearize(ctxt, addr, size, false, &linear);
777 if (rc != X86EMUL_CONTINUE)
779 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
783 * Prefetch the remaining bytes of the instruction without crossing page
784 * boundary if they are not in fetch_cache yet.
786 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
789 unsigned size, max_size;
790 unsigned long linear;
791 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
792 struct segmented_address addr = { .seg = VCPU_SREG_CS,
793 .ea = ctxt->eip + cur_size };
796 * We do not know exactly how many bytes will be needed, and
797 * __linearize is expensive, so fetch as much as possible. We
798 * just have to avoid going beyond the 15 byte limit, the end
799 * of the segment, or the end of the page.
801 * __linearize is called with size 0 so that it does not do any
802 * boundary check itself. Instead, we use max_size to check
805 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
807 if (unlikely(rc != X86EMUL_CONTINUE))
810 size = min_t(unsigned, 15UL ^ cur_size, max_size);
811 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
814 * One instruction can only straddle two pages,
815 * and one has been loaded at the beginning of
816 * x86_decode_insn. So, if not enough bytes
817 * still, we must have hit the 15-byte boundary.
819 if (unlikely(size < op_size))
820 return emulate_gp(ctxt, 0);
822 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
823 size, &ctxt->exception);
824 if (unlikely(rc != X86EMUL_CONTINUE))
826 ctxt->fetch.end += size;
827 return X86EMUL_CONTINUE;
830 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
833 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
835 if (unlikely(done_size < size))
836 return __do_insn_fetch_bytes(ctxt, size - done_size);
838 return X86EMUL_CONTINUE;
841 /* Fetch next part of the instruction being emulated. */
842 #define insn_fetch(_type, _ctxt) \
845 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
846 if (rc != X86EMUL_CONTINUE) \
848 ctxt->_eip += sizeof(_type); \
849 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
850 ctxt->fetch.ptr += sizeof(_type); \
854 #define insn_fetch_arr(_arr, _size, _ctxt) \
856 rc = do_insn_fetch_bytes(_ctxt, _size); \
857 if (rc != X86EMUL_CONTINUE) \
859 ctxt->_eip += (_size); \
860 memcpy(_arr, ctxt->fetch.ptr, _size); \
861 ctxt->fetch.ptr += (_size); \
865 * Given the 'reg' portion of a ModRM byte, and a register block, return a
866 * pointer into the block that addresses the relevant register.
867 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
869 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
873 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
875 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
876 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
878 p = reg_rmw(ctxt, modrm_reg);
882 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
883 struct segmented_address addr,
884 u16 *size, unsigned long *address, int op_bytes)
891 rc = segmented_read_std(ctxt, addr, size, 2);
892 if (rc != X86EMUL_CONTINUE)
895 rc = segmented_read_std(ctxt, addr, address, op_bytes);
909 FASTOP1SRC2(mul, mul_ex);
910 FASTOP1SRC2(imul, imul_ex);
911 FASTOP1SRC2EX(div, div_ex);
912 FASTOP1SRC2EX(idiv, idiv_ex);
941 FASTOP2R(cmp, cmp_r);
943 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
945 /* If src is zero, do not writeback, but update flags */
946 if (ctxt->src.val == 0)
947 ctxt->dst.type = OP_NONE;
948 return fastop(ctxt, em_bsf);
951 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
953 /* If src is zero, do not writeback, but update flags */
954 if (ctxt->src.val == 0)
955 ctxt->dst.type = OP_NONE;
956 return fastop(ctxt, em_bsr);
959 static u8 test_cc(unsigned int condition, unsigned long flags)
962 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
964 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
965 asm("push %[flags]; popf; call *%[fastop]"
966 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
970 static void fetch_register_operand(struct operand *op)
974 op->val = *(u8 *)op->addr.reg;
977 op->val = *(u16 *)op->addr.reg;
980 op->val = *(u32 *)op->addr.reg;
983 op->val = *(u64 *)op->addr.reg;
988 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
990 ctxt->ops->get_fpu(ctxt);
992 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
993 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
994 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
995 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
996 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
997 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
998 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
999 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1000 #ifdef CONFIG_X86_64
1001 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1002 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1003 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1004 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1005 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1006 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1007 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1008 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1012 ctxt->ops->put_fpu(ctxt);
1015 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1018 ctxt->ops->get_fpu(ctxt);
1020 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1021 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1022 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1023 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1024 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1025 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1026 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1027 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1028 #ifdef CONFIG_X86_64
1029 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1030 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1031 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1032 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1033 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1034 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1035 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1036 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1040 ctxt->ops->put_fpu(ctxt);
1043 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1045 ctxt->ops->get_fpu(ctxt);
1047 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1048 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1049 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1050 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1051 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1052 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1053 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1054 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1057 ctxt->ops->put_fpu(ctxt);
1060 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1062 ctxt->ops->get_fpu(ctxt);
1064 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1065 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1066 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1067 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1068 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1069 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1070 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1071 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1074 ctxt->ops->put_fpu(ctxt);
1077 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1079 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1080 return emulate_nm(ctxt);
1082 ctxt->ops->get_fpu(ctxt);
1083 asm volatile("fninit");
1084 ctxt->ops->put_fpu(ctxt);
1085 return X86EMUL_CONTINUE;
1088 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1092 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1093 return emulate_nm(ctxt);
1095 ctxt->ops->get_fpu(ctxt);
1096 asm volatile("fnstcw %0": "+m"(fcw));
1097 ctxt->ops->put_fpu(ctxt);
1099 ctxt->dst.val = fcw;
1101 return X86EMUL_CONTINUE;
1104 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1108 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1109 return emulate_nm(ctxt);
1111 ctxt->ops->get_fpu(ctxt);
1112 asm volatile("fnstsw %0": "+m"(fsw));
1113 ctxt->ops->put_fpu(ctxt);
1115 ctxt->dst.val = fsw;
1117 return X86EMUL_CONTINUE;
1120 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1123 unsigned reg = ctxt->modrm_reg;
1125 if (!(ctxt->d & ModRM))
1126 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1128 if (ctxt->d & Sse) {
1132 read_sse_reg(ctxt, &op->vec_val, reg);
1135 if (ctxt->d & Mmx) {
1144 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1145 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1147 fetch_register_operand(op);
1148 op->orig_val = op->val;
1151 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1153 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1154 ctxt->modrm_seg = VCPU_SREG_SS;
1157 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1161 int index_reg, base_reg, scale;
1162 int rc = X86EMUL_CONTINUE;
1165 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1166 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1167 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1169 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1170 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1171 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1172 ctxt->modrm_seg = VCPU_SREG_DS;
1174 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1176 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1177 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1179 if (ctxt->d & Sse) {
1182 op->addr.xmm = ctxt->modrm_rm;
1183 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1186 if (ctxt->d & Mmx) {
1189 op->addr.mm = ctxt->modrm_rm & 7;
1192 fetch_register_operand(op);
1198 if (ctxt->ad_bytes == 2) {
1199 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1200 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1201 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1202 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1204 /* 16-bit ModR/M decode. */
1205 switch (ctxt->modrm_mod) {
1207 if (ctxt->modrm_rm == 6)
1208 modrm_ea += insn_fetch(u16, ctxt);
1211 modrm_ea += insn_fetch(s8, ctxt);
1214 modrm_ea += insn_fetch(u16, ctxt);
1217 switch (ctxt->modrm_rm) {
1219 modrm_ea += bx + si;
1222 modrm_ea += bx + di;
1225 modrm_ea += bp + si;
1228 modrm_ea += bp + di;
1237 if (ctxt->modrm_mod != 0)
1244 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1245 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1246 ctxt->modrm_seg = VCPU_SREG_SS;
1247 modrm_ea = (u16)modrm_ea;
1249 /* 32/64-bit ModR/M decode. */
1250 if ((ctxt->modrm_rm & 7) == 4) {
1251 sib = insn_fetch(u8, ctxt);
1252 index_reg |= (sib >> 3) & 7;
1253 base_reg |= sib & 7;
1256 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1257 modrm_ea += insn_fetch(s32, ctxt);
1259 modrm_ea += reg_read(ctxt, base_reg);
1260 adjust_modrm_seg(ctxt, base_reg);
1261 /* Increment ESP on POP [ESP] */
1262 if ((ctxt->d & IncSP) &&
1263 base_reg == VCPU_REGS_RSP)
1264 modrm_ea += ctxt->op_bytes;
1267 modrm_ea += reg_read(ctxt, index_reg) << scale;
1268 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1269 modrm_ea += insn_fetch(s32, ctxt);
1270 if (ctxt->mode == X86EMUL_MODE_PROT64)
1271 ctxt->rip_relative = 1;
1273 base_reg = ctxt->modrm_rm;
1274 modrm_ea += reg_read(ctxt, base_reg);
1275 adjust_modrm_seg(ctxt, base_reg);
1277 switch (ctxt->modrm_mod) {
1279 modrm_ea += insn_fetch(s8, ctxt);
1282 modrm_ea += insn_fetch(s32, ctxt);
1286 op->addr.mem.ea = modrm_ea;
1287 if (ctxt->ad_bytes != 8)
1288 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1294 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1297 int rc = X86EMUL_CONTINUE;
1300 switch (ctxt->ad_bytes) {
1302 op->addr.mem.ea = insn_fetch(u16, ctxt);
1305 op->addr.mem.ea = insn_fetch(u32, ctxt);
1308 op->addr.mem.ea = insn_fetch(u64, ctxt);
1315 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1319 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1320 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1322 if (ctxt->src.bytes == 2)
1323 sv = (s16)ctxt->src.val & (s16)mask;
1324 else if (ctxt->src.bytes == 4)
1325 sv = (s32)ctxt->src.val & (s32)mask;
1327 sv = (s64)ctxt->src.val & (s64)mask;
1329 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1330 ctxt->dst.addr.mem.ea + (sv >> 3));
1333 /* only subword offset */
1334 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1337 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1338 unsigned long addr, void *dest, unsigned size)
1341 struct read_cache *mc = &ctxt->mem_read;
1343 if (mc->pos < mc->end)
1346 WARN_ON((mc->end + size) >= sizeof(mc->data));
1348 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1350 if (rc != X86EMUL_CONTINUE)
1356 memcpy(dest, mc->data + mc->pos, size);
1358 return X86EMUL_CONTINUE;
1361 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1362 struct segmented_address addr,
1369 rc = linearize(ctxt, addr, size, false, &linear);
1370 if (rc != X86EMUL_CONTINUE)
1372 return read_emulated(ctxt, linear, data, size);
1375 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1376 struct segmented_address addr,
1383 rc = linearize(ctxt, addr, size, true, &linear);
1384 if (rc != X86EMUL_CONTINUE)
1386 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1390 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1391 struct segmented_address addr,
1392 const void *orig_data, const void *data,
1398 rc = linearize(ctxt, addr, size, true, &linear);
1399 if (rc != X86EMUL_CONTINUE)
1401 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1402 size, &ctxt->exception);
1405 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1406 unsigned int size, unsigned short port,
1409 struct read_cache *rc = &ctxt->io_read;
1411 if (rc->pos == rc->end) { /* refill pio read ahead */
1412 unsigned int in_page, n;
1413 unsigned int count = ctxt->rep_prefix ?
1414 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1415 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1416 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1417 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1418 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1421 rc->pos = rc->end = 0;
1422 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1427 if (ctxt->rep_prefix && (ctxt->d & String) &&
1428 !(ctxt->eflags & X86_EFLAGS_DF)) {
1429 ctxt->dst.data = rc->data + rc->pos;
1430 ctxt->dst.type = OP_MEM_STR;
1431 ctxt->dst.count = (rc->end - rc->pos) / size;
1434 memcpy(dest, rc->data + rc->pos, size);
1440 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1441 u16 index, struct desc_struct *desc)
1446 ctxt->ops->get_idt(ctxt, &dt);
1448 if (dt.size < index * 8 + 7)
1449 return emulate_gp(ctxt, index << 3 | 0x2);
1451 addr = dt.address + index * 8;
1452 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1456 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1457 u16 selector, struct desc_ptr *dt)
1459 const struct x86_emulate_ops *ops = ctxt->ops;
1462 if (selector & 1 << 2) {
1463 struct desc_struct desc;
1466 memset (dt, 0, sizeof *dt);
1467 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1471 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1472 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1474 ops->get_gdt(ctxt, dt);
1477 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1478 u16 selector, ulong *desc_addr_p)
1481 u16 index = selector >> 3;
1484 get_descriptor_table_ptr(ctxt, selector, &dt);
1486 if (dt.size < index * 8 + 7)
1487 return emulate_gp(ctxt, selector & 0xfffc);
1489 addr = dt.address + index * 8;
1491 #ifdef CONFIG_X86_64
1492 if (addr >> 32 != 0) {
1495 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1496 if (!(efer & EFER_LMA))
1501 *desc_addr_p = addr;
1502 return X86EMUL_CONTINUE;
1505 /* allowed just for 8 bytes segments */
1506 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1507 u16 selector, struct desc_struct *desc,
1512 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1513 if (rc != X86EMUL_CONTINUE)
1516 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1520 /* allowed just for 8 bytes segments */
1521 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1522 u16 selector, struct desc_struct *desc)
1527 rc = get_descriptor_ptr(ctxt, selector, &addr);
1528 if (rc != X86EMUL_CONTINUE)
1531 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1535 /* Does not support long mode */
1536 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1537 u16 selector, int seg, u8 cpl,
1538 enum x86_transfer_type transfer,
1539 struct desc_struct *desc)
1541 struct desc_struct seg_desc, old_desc;
1543 unsigned err_vec = GP_VECTOR;
1545 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1551 memset(&seg_desc, 0, sizeof seg_desc);
1553 if (ctxt->mode == X86EMUL_MODE_REAL) {
1554 /* set real mode segment descriptor (keep limit etc. for
1556 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1557 set_desc_base(&seg_desc, selector << 4);
1559 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1560 /* VM86 needs a clean new segment descriptor */
1561 set_desc_base(&seg_desc, selector << 4);
1562 set_desc_limit(&seg_desc, 0xffff);
1572 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1573 if ((seg == VCPU_SREG_CS
1574 || (seg == VCPU_SREG_SS
1575 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1576 || seg == VCPU_SREG_TR)
1580 /* TR should be in GDT only */
1581 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1584 if (null_selector) /* for NULL selector skip all following checks */
1587 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1588 if (ret != X86EMUL_CONTINUE)
1591 err_code = selector & 0xfffc;
1592 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1595 /* can't load system descriptor into segment selector */
1596 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1597 if (transfer == X86_TRANSFER_CALL_JMP)
1598 return X86EMUL_UNHANDLEABLE;
1603 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1612 * segment is not a writable data segment or segment
1613 * selector's RPL != CPL or segment selector's RPL != CPL
1615 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1619 if (!(seg_desc.type & 8))
1622 if (seg_desc.type & 4) {
1628 if (rpl > cpl || dpl != cpl)
1631 /* in long-mode d/b must be clear if l is set */
1632 if (seg_desc.d && seg_desc.l) {
1635 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1636 if (efer & EFER_LMA)
1640 /* CS(RPL) <- CPL */
1641 selector = (selector & 0xfffc) | cpl;
1644 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1646 old_desc = seg_desc;
1647 seg_desc.type |= 2; /* busy */
1648 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1649 sizeof(seg_desc), &ctxt->exception);
1650 if (ret != X86EMUL_CONTINUE)
1653 case VCPU_SREG_LDTR:
1654 if (seg_desc.s || seg_desc.type != 2)
1657 default: /* DS, ES, FS, or GS */
1659 * segment is not a data or readable code segment or
1660 * ((segment is a data or nonconforming code segment)
1661 * and (both RPL and CPL > DPL))
1663 if ((seg_desc.type & 0xa) == 0x8 ||
1664 (((seg_desc.type & 0xc) != 0xc) &&
1665 (rpl > dpl && cpl > dpl)))
1671 /* mark segment as accessed */
1672 if (!(seg_desc.type & 1)) {
1674 ret = write_segment_descriptor(ctxt, selector,
1676 if (ret != X86EMUL_CONTINUE)
1679 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1680 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1681 sizeof(base3), &ctxt->exception);
1682 if (ret != X86EMUL_CONTINUE)
1684 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1685 ((u64)base3 << 32)))
1686 return emulate_gp(ctxt, 0);
1689 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1692 return X86EMUL_CONTINUE;
1694 return emulate_exception(ctxt, err_vec, err_code, true);
1697 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1698 u16 selector, int seg)
1700 u8 cpl = ctxt->ops->cpl(ctxt);
1701 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1702 X86_TRANSFER_NONE, NULL);
1705 static void write_register_operand(struct operand *op)
1707 return assign_register(op->addr.reg, op->val, op->bytes);
1710 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1714 write_register_operand(op);
1717 if (ctxt->lock_prefix)
1718 return segmented_cmpxchg(ctxt,
1724 return segmented_write(ctxt,
1730 return segmented_write(ctxt,
1733 op->bytes * op->count);
1736 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1739 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1747 return X86EMUL_CONTINUE;
1750 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1752 struct segmented_address addr;
1754 rsp_increment(ctxt, -bytes);
1755 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1756 addr.seg = VCPU_SREG_SS;
1758 return segmented_write(ctxt, addr, data, bytes);
1761 static int em_push(struct x86_emulate_ctxt *ctxt)
1763 /* Disable writeback. */
1764 ctxt->dst.type = OP_NONE;
1765 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1768 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1769 void *dest, int len)
1772 struct segmented_address addr;
1774 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1775 addr.seg = VCPU_SREG_SS;
1776 rc = segmented_read(ctxt, addr, dest, len);
1777 if (rc != X86EMUL_CONTINUE)
1780 rsp_increment(ctxt, len);
1784 static int em_pop(struct x86_emulate_ctxt *ctxt)
1786 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1789 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1790 void *dest, int len)
1793 unsigned long val, change_mask;
1794 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1795 int cpl = ctxt->ops->cpl(ctxt);
1797 rc = emulate_pop(ctxt, &val, len);
1798 if (rc != X86EMUL_CONTINUE)
1801 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1802 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1803 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1804 X86_EFLAGS_AC | X86_EFLAGS_ID;
1806 switch(ctxt->mode) {
1807 case X86EMUL_MODE_PROT64:
1808 case X86EMUL_MODE_PROT32:
1809 case X86EMUL_MODE_PROT16:
1811 change_mask |= X86_EFLAGS_IOPL;
1813 change_mask |= X86_EFLAGS_IF;
1815 case X86EMUL_MODE_VM86:
1817 return emulate_gp(ctxt, 0);
1818 change_mask |= X86_EFLAGS_IF;
1820 default: /* real mode */
1821 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1825 *(unsigned long *)dest =
1826 (ctxt->eflags & ~change_mask) | (val & change_mask);
1831 static int em_popf(struct x86_emulate_ctxt *ctxt)
1833 ctxt->dst.type = OP_REG;
1834 ctxt->dst.addr.reg = &ctxt->eflags;
1835 ctxt->dst.bytes = ctxt->op_bytes;
1836 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1839 static int em_enter(struct x86_emulate_ctxt *ctxt)
1842 unsigned frame_size = ctxt->src.val;
1843 unsigned nesting_level = ctxt->src2.val & 31;
1847 return X86EMUL_UNHANDLEABLE;
1849 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1850 rc = push(ctxt, &rbp, stack_size(ctxt));
1851 if (rc != X86EMUL_CONTINUE)
1853 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1855 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1856 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1858 return X86EMUL_CONTINUE;
1861 static int em_leave(struct x86_emulate_ctxt *ctxt)
1863 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1865 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1868 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1870 int seg = ctxt->src2.val;
1872 ctxt->src.val = get_segment_selector(ctxt, seg);
1873 if (ctxt->op_bytes == 4) {
1874 rsp_increment(ctxt, -2);
1878 return em_push(ctxt);
1881 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1883 int seg = ctxt->src2.val;
1884 unsigned long selector;
1887 rc = emulate_pop(ctxt, &selector, 2);
1888 if (rc != X86EMUL_CONTINUE)
1891 if (ctxt->modrm_reg == VCPU_SREG_SS)
1892 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1893 if (ctxt->op_bytes > 2)
1894 rsp_increment(ctxt, ctxt->op_bytes - 2);
1896 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1900 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1902 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1903 int rc = X86EMUL_CONTINUE;
1904 int reg = VCPU_REGS_RAX;
1906 while (reg <= VCPU_REGS_RDI) {
1907 (reg == VCPU_REGS_RSP) ?
1908 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1911 if (rc != X86EMUL_CONTINUE)
1920 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1922 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1923 return em_push(ctxt);
1926 static int em_popa(struct x86_emulate_ctxt *ctxt)
1928 int rc = X86EMUL_CONTINUE;
1929 int reg = VCPU_REGS_RDI;
1932 while (reg >= VCPU_REGS_RAX) {
1933 if (reg == VCPU_REGS_RSP) {
1934 rsp_increment(ctxt, ctxt->op_bytes);
1938 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1939 if (rc != X86EMUL_CONTINUE)
1941 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1947 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1949 const struct x86_emulate_ops *ops = ctxt->ops;
1956 /* TODO: Add limit checks */
1957 ctxt->src.val = ctxt->eflags;
1959 if (rc != X86EMUL_CONTINUE)
1962 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
1964 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1966 if (rc != X86EMUL_CONTINUE)
1969 ctxt->src.val = ctxt->_eip;
1971 if (rc != X86EMUL_CONTINUE)
1974 ops->get_idt(ctxt, &dt);
1976 eip_addr = dt.address + (irq << 2);
1977 cs_addr = dt.address + (irq << 2) + 2;
1979 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1980 if (rc != X86EMUL_CONTINUE)
1983 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1984 if (rc != X86EMUL_CONTINUE)
1987 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1988 if (rc != X86EMUL_CONTINUE)
1996 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2000 invalidate_registers(ctxt);
2001 rc = __emulate_int_real(ctxt, irq);
2002 if (rc == X86EMUL_CONTINUE)
2003 writeback_registers(ctxt);
2007 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2009 switch(ctxt->mode) {
2010 case X86EMUL_MODE_REAL:
2011 return __emulate_int_real(ctxt, irq);
2012 case X86EMUL_MODE_VM86:
2013 case X86EMUL_MODE_PROT16:
2014 case X86EMUL_MODE_PROT32:
2015 case X86EMUL_MODE_PROT64:
2017 /* Protected mode interrupts unimplemented yet */
2018 return X86EMUL_UNHANDLEABLE;
2022 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2024 int rc = X86EMUL_CONTINUE;
2025 unsigned long temp_eip = 0;
2026 unsigned long temp_eflags = 0;
2027 unsigned long cs = 0;
2028 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2029 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2030 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2031 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2032 X86_EFLAGS_AC | X86_EFLAGS_ID |
2034 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2037 /* TODO: Add stack limit check */
2039 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2041 if (rc != X86EMUL_CONTINUE)
2044 if (temp_eip & ~0xffff)
2045 return emulate_gp(ctxt, 0);
2047 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2049 if (rc != X86EMUL_CONTINUE)
2052 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2054 if (rc != X86EMUL_CONTINUE)
2057 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2059 if (rc != X86EMUL_CONTINUE)
2062 ctxt->_eip = temp_eip;
2064 if (ctxt->op_bytes == 4)
2065 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2066 else if (ctxt->op_bytes == 2) {
2067 ctxt->eflags &= ~0xffff;
2068 ctxt->eflags |= temp_eflags;
2071 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2072 ctxt->eflags |= X86_EFLAGS_FIXED;
2073 ctxt->ops->set_nmi_mask(ctxt, false);
2078 static int em_iret(struct x86_emulate_ctxt *ctxt)
2080 switch(ctxt->mode) {
2081 case X86EMUL_MODE_REAL:
2082 return emulate_iret_real(ctxt);
2083 case X86EMUL_MODE_VM86:
2084 case X86EMUL_MODE_PROT16:
2085 case X86EMUL_MODE_PROT32:
2086 case X86EMUL_MODE_PROT64:
2088 /* iret from protected mode unimplemented yet */
2089 return X86EMUL_UNHANDLEABLE;
2093 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2096 unsigned short sel, old_sel;
2097 struct desc_struct old_desc, new_desc;
2098 const struct x86_emulate_ops *ops = ctxt->ops;
2099 u8 cpl = ctxt->ops->cpl(ctxt);
2101 /* Assignment of RIP may only fail in 64-bit mode */
2102 if (ctxt->mode == X86EMUL_MODE_PROT64)
2103 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2106 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2108 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2109 X86_TRANSFER_CALL_JMP,
2111 if (rc != X86EMUL_CONTINUE)
2114 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2115 if (rc != X86EMUL_CONTINUE) {
2116 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2117 /* assigning eip failed; restore the old cs */
2118 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2124 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2126 return assign_eip_near(ctxt, ctxt->src.val);
2129 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2134 old_eip = ctxt->_eip;
2135 rc = assign_eip_near(ctxt, ctxt->src.val);
2136 if (rc != X86EMUL_CONTINUE)
2138 ctxt->src.val = old_eip;
2143 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2145 u64 old = ctxt->dst.orig_val64;
2147 if (ctxt->dst.bytes == 16)
2148 return X86EMUL_UNHANDLEABLE;
2150 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2151 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2152 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2153 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2154 ctxt->eflags &= ~X86_EFLAGS_ZF;
2156 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2157 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2159 ctxt->eflags |= X86_EFLAGS_ZF;
2161 return X86EMUL_CONTINUE;
2164 static int em_ret(struct x86_emulate_ctxt *ctxt)
2169 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2170 if (rc != X86EMUL_CONTINUE)
2173 return assign_eip_near(ctxt, eip);
2176 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2179 unsigned long eip, cs;
2181 int cpl = ctxt->ops->cpl(ctxt);
2182 struct desc_struct old_desc, new_desc;
2183 const struct x86_emulate_ops *ops = ctxt->ops;
2185 if (ctxt->mode == X86EMUL_MODE_PROT64)
2186 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2189 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2190 if (rc != X86EMUL_CONTINUE)
2192 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2193 if (rc != X86EMUL_CONTINUE)
2195 /* Outer-privilege level return is not implemented */
2196 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2197 return X86EMUL_UNHANDLEABLE;
2198 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2201 if (rc != X86EMUL_CONTINUE)
2203 rc = assign_eip_far(ctxt, eip, &new_desc);
2204 if (rc != X86EMUL_CONTINUE) {
2205 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2206 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2211 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2215 rc = em_ret_far(ctxt);
2216 if (rc != X86EMUL_CONTINUE)
2218 rsp_increment(ctxt, ctxt->src.val);
2219 return X86EMUL_CONTINUE;
2222 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2224 /* Save real source value, then compare EAX against destination. */
2225 ctxt->dst.orig_val = ctxt->dst.val;
2226 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2227 ctxt->src.orig_val = ctxt->src.val;
2228 ctxt->src.val = ctxt->dst.orig_val;
2229 fastop(ctxt, em_cmp);
2231 if (ctxt->eflags & X86_EFLAGS_ZF) {
2232 /* Success: write back to memory; no update of EAX */
2233 ctxt->src.type = OP_NONE;
2234 ctxt->dst.val = ctxt->src.orig_val;
2236 /* Failure: write the value we saw to EAX. */
2237 ctxt->src.type = OP_REG;
2238 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2239 ctxt->src.val = ctxt->dst.orig_val;
2240 /* Create write-cycle to dest by writing the same value */
2241 ctxt->dst.val = ctxt->dst.orig_val;
2243 return X86EMUL_CONTINUE;
2246 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2248 int seg = ctxt->src2.val;
2252 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2254 rc = load_segment_descriptor(ctxt, sel, seg);
2255 if (rc != X86EMUL_CONTINUE)
2258 ctxt->dst.val = ctxt->src.val;
2262 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2264 if ((ctxt->emul_flags & X86EMUL_SMM_MASK) == 0)
2265 return emulate_ud(ctxt);
2267 return X86EMUL_UNHANDLEABLE;
2271 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2272 struct desc_struct *cs, struct desc_struct *ss)
2274 cs->l = 0; /* will be adjusted later */
2275 set_desc_base(cs, 0); /* flat segment */
2276 cs->g = 1; /* 4kb granularity */
2277 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2278 cs->type = 0x0b; /* Read, Execute, Accessed */
2280 cs->dpl = 0; /* will be adjusted later */
2285 set_desc_base(ss, 0); /* flat segment */
2286 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2287 ss->g = 1; /* 4kb granularity */
2289 ss->type = 0x03; /* Read/Write, Accessed */
2290 ss->d = 1; /* 32bit stack segment */
2297 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2299 u32 eax, ebx, ecx, edx;
2302 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2303 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2304 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2305 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2308 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2310 const struct x86_emulate_ops *ops = ctxt->ops;
2311 u32 eax, ebx, ecx, edx;
2314 * syscall should always be enabled in longmode - so only become
2315 * vendor specific (cpuid) if other modes are active...
2317 if (ctxt->mode == X86EMUL_MODE_PROT64)
2322 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2324 * Intel ("GenuineIntel")
2325 * remark: Intel CPUs only support "syscall" in 64bit
2326 * longmode. Also an 64bit guest with a
2327 * 32bit compat-app running will #UD !! While this
2328 * behaviour can be fixed (by emulating) into AMD
2329 * response - CPUs of AMD can't behave like Intel.
2331 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2332 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2333 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2336 /* AMD ("AuthenticAMD") */
2337 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2338 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2339 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2342 /* AMD ("AMDisbetter!") */
2343 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2344 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2345 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2348 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2352 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2354 const struct x86_emulate_ops *ops = ctxt->ops;
2355 struct desc_struct cs, ss;
2360 /* syscall is not available in real mode */
2361 if (ctxt->mode == X86EMUL_MODE_REAL ||
2362 ctxt->mode == X86EMUL_MODE_VM86)
2363 return emulate_ud(ctxt);
2365 if (!(em_syscall_is_enabled(ctxt)))
2366 return emulate_ud(ctxt);
2368 ops->get_msr(ctxt, MSR_EFER, &efer);
2369 setup_syscalls_segments(ctxt, &cs, &ss);
2371 if (!(efer & EFER_SCE))
2372 return emulate_ud(ctxt);
2374 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2376 cs_sel = (u16)(msr_data & 0xfffc);
2377 ss_sel = (u16)(msr_data + 8);
2379 if (efer & EFER_LMA) {
2383 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2384 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2386 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2387 if (efer & EFER_LMA) {
2388 #ifdef CONFIG_X86_64
2389 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2392 ctxt->mode == X86EMUL_MODE_PROT64 ?
2393 MSR_LSTAR : MSR_CSTAR, &msr_data);
2394 ctxt->_eip = msr_data;
2396 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2397 ctxt->eflags &= ~msr_data;
2398 ctxt->eflags |= X86_EFLAGS_FIXED;
2402 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2403 ctxt->_eip = (u32)msr_data;
2405 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2408 return X86EMUL_CONTINUE;
2411 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2413 const struct x86_emulate_ops *ops = ctxt->ops;
2414 struct desc_struct cs, ss;
2419 ops->get_msr(ctxt, MSR_EFER, &efer);
2420 /* inject #GP if in real mode */
2421 if (ctxt->mode == X86EMUL_MODE_REAL)
2422 return emulate_gp(ctxt, 0);
2425 * Not recognized on AMD in compat mode (but is recognized in legacy
2428 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2429 && !vendor_intel(ctxt))
2430 return emulate_ud(ctxt);
2432 /* sysenter/sysexit have not been tested in 64bit mode. */
2433 if (ctxt->mode == X86EMUL_MODE_PROT64)
2434 return X86EMUL_UNHANDLEABLE;
2436 setup_syscalls_segments(ctxt, &cs, &ss);
2438 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2439 if ((msr_data & 0xfffc) == 0x0)
2440 return emulate_gp(ctxt, 0);
2442 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2443 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2444 ss_sel = cs_sel + 8;
2445 if (efer & EFER_LMA) {
2450 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2451 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2453 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2454 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2456 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2457 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2460 return X86EMUL_CONTINUE;
2463 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2465 const struct x86_emulate_ops *ops = ctxt->ops;
2466 struct desc_struct cs, ss;
2467 u64 msr_data, rcx, rdx;
2469 u16 cs_sel = 0, ss_sel = 0;
2471 /* inject #GP if in real mode or Virtual 8086 mode */
2472 if (ctxt->mode == X86EMUL_MODE_REAL ||
2473 ctxt->mode == X86EMUL_MODE_VM86)
2474 return emulate_gp(ctxt, 0);
2476 setup_syscalls_segments(ctxt, &cs, &ss);
2478 if ((ctxt->rex_prefix & 0x8) != 0x0)
2479 usermode = X86EMUL_MODE_PROT64;
2481 usermode = X86EMUL_MODE_PROT32;
2483 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2484 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2488 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2490 case X86EMUL_MODE_PROT32:
2491 cs_sel = (u16)(msr_data + 16);
2492 if ((msr_data & 0xfffc) == 0x0)
2493 return emulate_gp(ctxt, 0);
2494 ss_sel = (u16)(msr_data + 24);
2498 case X86EMUL_MODE_PROT64:
2499 cs_sel = (u16)(msr_data + 32);
2500 if (msr_data == 0x0)
2501 return emulate_gp(ctxt, 0);
2502 ss_sel = cs_sel + 8;
2505 if (is_noncanonical_address(rcx) ||
2506 is_noncanonical_address(rdx))
2507 return emulate_gp(ctxt, 0);
2510 cs_sel |= SEGMENT_RPL_MASK;
2511 ss_sel |= SEGMENT_RPL_MASK;
2513 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2514 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2517 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2519 return X86EMUL_CONTINUE;
2522 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2525 if (ctxt->mode == X86EMUL_MODE_REAL)
2527 if (ctxt->mode == X86EMUL_MODE_VM86)
2529 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2530 return ctxt->ops->cpl(ctxt) > iopl;
2533 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2536 const struct x86_emulate_ops *ops = ctxt->ops;
2537 struct desc_struct tr_seg;
2540 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2541 unsigned mask = (1 << len) - 1;
2544 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2547 if (desc_limit_scaled(&tr_seg) < 103)
2549 base = get_desc_base(&tr_seg);
2550 #ifdef CONFIG_X86_64
2551 base |= ((u64)base3) << 32;
2553 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2554 if (r != X86EMUL_CONTINUE)
2556 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2558 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2559 if (r != X86EMUL_CONTINUE)
2561 if ((perm >> bit_idx) & mask)
2566 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2572 if (emulator_bad_iopl(ctxt))
2573 if (!emulator_io_port_access_allowed(ctxt, port, len))
2576 ctxt->perm_ok = true;
2581 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2584 * Intel CPUs mask the counter and pointers in quite strange
2585 * manner when ECX is zero due to REP-string optimizations.
2587 #ifdef CONFIG_X86_64
2588 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2591 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2594 case 0xa4: /* movsb */
2595 case 0xa5: /* movsd/w */
2596 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2598 case 0xaa: /* stosb */
2599 case 0xab: /* stosd/w */
2600 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2605 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2606 struct tss_segment_16 *tss)
2608 tss->ip = ctxt->_eip;
2609 tss->flag = ctxt->eflags;
2610 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2611 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2612 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2613 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2614 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2615 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2616 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2617 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2619 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2620 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2621 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2622 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2623 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2626 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2627 struct tss_segment_16 *tss)
2632 ctxt->_eip = tss->ip;
2633 ctxt->eflags = tss->flag | 2;
2634 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2635 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2636 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2637 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2638 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2639 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2640 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2641 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2644 * SDM says that segment selectors are loaded before segment
2647 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2648 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2649 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2650 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2651 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2656 * Now load segment descriptors. If fault happens at this stage
2657 * it is handled in a context of new task
2659 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2660 X86_TRANSFER_TASK_SWITCH, NULL);
2661 if (ret != X86EMUL_CONTINUE)
2663 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2664 X86_TRANSFER_TASK_SWITCH, NULL);
2665 if (ret != X86EMUL_CONTINUE)
2667 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2668 X86_TRANSFER_TASK_SWITCH, NULL);
2669 if (ret != X86EMUL_CONTINUE)
2671 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2672 X86_TRANSFER_TASK_SWITCH, NULL);
2673 if (ret != X86EMUL_CONTINUE)
2675 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2676 X86_TRANSFER_TASK_SWITCH, NULL);
2677 if (ret != X86EMUL_CONTINUE)
2680 return X86EMUL_CONTINUE;
2683 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2684 u16 tss_selector, u16 old_tss_sel,
2685 ulong old_tss_base, struct desc_struct *new_desc)
2687 const struct x86_emulate_ops *ops = ctxt->ops;
2688 struct tss_segment_16 tss_seg;
2690 u32 new_tss_base = get_desc_base(new_desc);
2692 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2694 if (ret != X86EMUL_CONTINUE)
2697 save_state_to_tss16(ctxt, &tss_seg);
2699 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2701 if (ret != X86EMUL_CONTINUE)
2704 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2706 if (ret != X86EMUL_CONTINUE)
2709 if (old_tss_sel != 0xffff) {
2710 tss_seg.prev_task_link = old_tss_sel;
2712 ret = ops->write_std(ctxt, new_tss_base,
2713 &tss_seg.prev_task_link,
2714 sizeof tss_seg.prev_task_link,
2716 if (ret != X86EMUL_CONTINUE)
2720 return load_state_from_tss16(ctxt, &tss_seg);
2723 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2724 struct tss_segment_32 *tss)
2726 /* CR3 and ldt selector are not saved intentionally */
2727 tss->eip = ctxt->_eip;
2728 tss->eflags = ctxt->eflags;
2729 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2730 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2731 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2732 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2733 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2734 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2735 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2736 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2738 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2739 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2740 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2741 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2742 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2743 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2746 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2747 struct tss_segment_32 *tss)
2752 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2753 return emulate_gp(ctxt, 0);
2754 ctxt->_eip = tss->eip;
2755 ctxt->eflags = tss->eflags | 2;
2757 /* General purpose registers */
2758 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2759 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2760 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2761 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2762 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2763 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2764 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2765 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2768 * SDM says that segment selectors are loaded before segment
2769 * descriptors. This is important because CPL checks will
2772 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2773 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2774 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2775 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2776 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2777 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2778 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2781 * If we're switching between Protected Mode and VM86, we need to make
2782 * sure to update the mode before loading the segment descriptors so
2783 * that the selectors are interpreted correctly.
2785 if (ctxt->eflags & X86_EFLAGS_VM) {
2786 ctxt->mode = X86EMUL_MODE_VM86;
2789 ctxt->mode = X86EMUL_MODE_PROT32;
2794 * Now load segment descriptors. If fault happenes at this stage
2795 * it is handled in a context of new task
2797 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2798 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2799 if (ret != X86EMUL_CONTINUE)
2801 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2802 X86_TRANSFER_TASK_SWITCH, NULL);
2803 if (ret != X86EMUL_CONTINUE)
2805 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2806 X86_TRANSFER_TASK_SWITCH, NULL);
2807 if (ret != X86EMUL_CONTINUE)
2809 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2810 X86_TRANSFER_TASK_SWITCH, NULL);
2811 if (ret != X86EMUL_CONTINUE)
2813 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2814 X86_TRANSFER_TASK_SWITCH, NULL);
2815 if (ret != X86EMUL_CONTINUE)
2817 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2818 X86_TRANSFER_TASK_SWITCH, NULL);
2819 if (ret != X86EMUL_CONTINUE)
2821 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2822 X86_TRANSFER_TASK_SWITCH, NULL);
2827 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2828 u16 tss_selector, u16 old_tss_sel,
2829 ulong old_tss_base, struct desc_struct *new_desc)
2831 const struct x86_emulate_ops *ops = ctxt->ops;
2832 struct tss_segment_32 tss_seg;
2834 u32 new_tss_base = get_desc_base(new_desc);
2835 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2836 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2838 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2840 if (ret != X86EMUL_CONTINUE)
2843 save_state_to_tss32(ctxt, &tss_seg);
2845 /* Only GP registers and segment selectors are saved */
2846 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2847 ldt_sel_offset - eip_offset, &ctxt->exception);
2848 if (ret != X86EMUL_CONTINUE)
2851 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2853 if (ret != X86EMUL_CONTINUE)
2856 if (old_tss_sel != 0xffff) {
2857 tss_seg.prev_task_link = old_tss_sel;
2859 ret = ops->write_std(ctxt, new_tss_base,
2860 &tss_seg.prev_task_link,
2861 sizeof tss_seg.prev_task_link,
2863 if (ret != X86EMUL_CONTINUE)
2867 return load_state_from_tss32(ctxt, &tss_seg);
2870 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2871 u16 tss_selector, int idt_index, int reason,
2872 bool has_error_code, u32 error_code)
2874 const struct x86_emulate_ops *ops = ctxt->ops;
2875 struct desc_struct curr_tss_desc, next_tss_desc;
2877 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2878 ulong old_tss_base =
2879 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2881 ulong desc_addr, dr7;
2883 /* FIXME: old_tss_base == ~0 ? */
2885 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2886 if (ret != X86EMUL_CONTINUE)
2888 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2889 if (ret != X86EMUL_CONTINUE)
2892 /* FIXME: check that next_tss_desc is tss */
2895 * Check privileges. The three cases are task switch caused by...
2897 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2898 * 2. Exception/IRQ/iret: No check is performed
2899 * 3. jmp/call to TSS/task-gate: No check is performed since the
2900 * hardware checks it before exiting.
2902 if (reason == TASK_SWITCH_GATE) {
2903 if (idt_index != -1) {
2904 /* Software interrupts */
2905 struct desc_struct task_gate_desc;
2908 ret = read_interrupt_descriptor(ctxt, idt_index,
2910 if (ret != X86EMUL_CONTINUE)
2913 dpl = task_gate_desc.dpl;
2914 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2915 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2919 desc_limit = desc_limit_scaled(&next_tss_desc);
2920 if (!next_tss_desc.p ||
2921 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2922 desc_limit < 0x2b)) {
2923 return emulate_ts(ctxt, tss_selector & 0xfffc);
2926 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2927 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2928 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2931 if (reason == TASK_SWITCH_IRET)
2932 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2934 /* set back link to prev task only if NT bit is set in eflags
2935 note that old_tss_sel is not used after this point */
2936 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2937 old_tss_sel = 0xffff;
2939 if (next_tss_desc.type & 8)
2940 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2941 old_tss_base, &next_tss_desc);
2943 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2944 old_tss_base, &next_tss_desc);
2945 if (ret != X86EMUL_CONTINUE)
2948 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2949 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2951 if (reason != TASK_SWITCH_IRET) {
2952 next_tss_desc.type |= (1 << 1); /* set busy flag */
2953 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2956 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2957 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2959 if (has_error_code) {
2960 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2961 ctxt->lock_prefix = 0;
2962 ctxt->src.val = (unsigned long) error_code;
2963 ret = em_push(ctxt);
2966 ops->get_dr(ctxt, 7, &dr7);
2967 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
2972 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2973 u16 tss_selector, int idt_index, int reason,
2974 bool has_error_code, u32 error_code)
2978 invalidate_registers(ctxt);
2979 ctxt->_eip = ctxt->eip;
2980 ctxt->dst.type = OP_NONE;
2982 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2983 has_error_code, error_code);
2985 if (rc == X86EMUL_CONTINUE) {
2986 ctxt->eip = ctxt->_eip;
2987 writeback_registers(ctxt);
2990 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2993 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2996 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
2998 register_address_increment(ctxt, reg, df * op->bytes);
2999 op->addr.mem.ea = register_address(ctxt, reg);
3002 static int em_das(struct x86_emulate_ctxt *ctxt)
3005 bool af, cf, old_cf;
3007 cf = ctxt->eflags & X86_EFLAGS_CF;
3013 af = ctxt->eflags & X86_EFLAGS_AF;
3014 if ((al & 0x0f) > 9 || af) {
3016 cf = old_cf | (al >= 250);
3021 if (old_al > 0x99 || old_cf) {
3027 /* Set PF, ZF, SF */
3028 ctxt->src.type = OP_IMM;
3030 ctxt->src.bytes = 1;
3031 fastop(ctxt, em_or);
3032 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3034 ctxt->eflags |= X86_EFLAGS_CF;
3036 ctxt->eflags |= X86_EFLAGS_AF;
3037 return X86EMUL_CONTINUE;
3040 static int em_aam(struct x86_emulate_ctxt *ctxt)
3044 if (ctxt->src.val == 0)
3045 return emulate_de(ctxt);
3047 al = ctxt->dst.val & 0xff;
3048 ah = al / ctxt->src.val;
3049 al %= ctxt->src.val;
3051 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3053 /* Set PF, ZF, SF */
3054 ctxt->src.type = OP_IMM;
3056 ctxt->src.bytes = 1;
3057 fastop(ctxt, em_or);
3059 return X86EMUL_CONTINUE;
3062 static int em_aad(struct x86_emulate_ctxt *ctxt)
3064 u8 al = ctxt->dst.val & 0xff;
3065 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3067 al = (al + (ah * ctxt->src.val)) & 0xff;
3069 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3071 /* Set PF, ZF, SF */
3072 ctxt->src.type = OP_IMM;
3074 ctxt->src.bytes = 1;
3075 fastop(ctxt, em_or);
3077 return X86EMUL_CONTINUE;
3080 static int em_call(struct x86_emulate_ctxt *ctxt)
3083 long rel = ctxt->src.val;
3085 ctxt->src.val = (unsigned long)ctxt->_eip;
3086 rc = jmp_rel(ctxt, rel);
3087 if (rc != X86EMUL_CONTINUE)
3089 return em_push(ctxt);
3092 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3097 struct desc_struct old_desc, new_desc;
3098 const struct x86_emulate_ops *ops = ctxt->ops;
3099 int cpl = ctxt->ops->cpl(ctxt);
3100 enum x86emul_mode prev_mode = ctxt->mode;
3102 old_eip = ctxt->_eip;
3103 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3105 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3106 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3107 X86_TRANSFER_CALL_JMP, &new_desc);
3108 if (rc != X86EMUL_CONTINUE)
3111 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3112 if (rc != X86EMUL_CONTINUE)
3115 ctxt->src.val = old_cs;
3117 if (rc != X86EMUL_CONTINUE)
3120 ctxt->src.val = old_eip;
3122 /* If we failed, we tainted the memory, but the very least we should
3124 if (rc != X86EMUL_CONTINUE) {
3125 pr_warn_once("faulting far call emulation tainted memory\n");
3130 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3131 ctxt->mode = prev_mode;
3136 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3141 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3142 if (rc != X86EMUL_CONTINUE)
3144 rc = assign_eip_near(ctxt, eip);
3145 if (rc != X86EMUL_CONTINUE)
3147 rsp_increment(ctxt, ctxt->src.val);
3148 return X86EMUL_CONTINUE;
3151 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3153 /* Write back the register source. */
3154 ctxt->src.val = ctxt->dst.val;
3155 write_register_operand(&ctxt->src);
3157 /* Write back the memory destination with implicit LOCK prefix. */
3158 ctxt->dst.val = ctxt->src.orig_val;
3159 ctxt->lock_prefix = 1;
3160 return X86EMUL_CONTINUE;
3163 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3165 ctxt->dst.val = ctxt->src2.val;
3166 return fastop(ctxt, em_imul);
3169 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3171 ctxt->dst.type = OP_REG;
3172 ctxt->dst.bytes = ctxt->src.bytes;
3173 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3174 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3176 return X86EMUL_CONTINUE;
3179 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3183 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3184 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3185 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3186 return X86EMUL_CONTINUE;
3189 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3193 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3194 return emulate_gp(ctxt, 0);
3195 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3196 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3197 return X86EMUL_CONTINUE;
3200 static int em_mov(struct x86_emulate_ctxt *ctxt)
3202 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3203 return X86EMUL_CONTINUE;
3206 #define FFL(x) bit(X86_FEATURE_##x)
3208 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3210 u32 ebx, ecx, edx, eax = 1;
3214 * Check MOVBE is set in the guest-visible CPUID leaf.
3216 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3217 if (!(ecx & FFL(MOVBE)))
3218 return emulate_ud(ctxt);
3220 switch (ctxt->op_bytes) {
3223 * From MOVBE definition: "...When the operand size is 16 bits,
3224 * the upper word of the destination register remains unchanged
3227 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3228 * rules so we have to do the operation almost per hand.
3230 tmp = (u16)ctxt->src.val;
3231 ctxt->dst.val &= ~0xffffUL;
3232 ctxt->dst.val |= (unsigned long)swab16(tmp);
3235 ctxt->dst.val = swab32((u32)ctxt->src.val);
3238 ctxt->dst.val = swab64(ctxt->src.val);
3243 return X86EMUL_CONTINUE;
3246 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3248 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3249 return emulate_gp(ctxt, 0);
3251 /* Disable writeback. */
3252 ctxt->dst.type = OP_NONE;
3253 return X86EMUL_CONTINUE;
3256 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3260 if (ctxt->mode == X86EMUL_MODE_PROT64)
3261 val = ctxt->src.val & ~0ULL;
3263 val = ctxt->src.val & ~0U;
3265 /* #UD condition is already handled. */
3266 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3267 return emulate_gp(ctxt, 0);
3269 /* Disable writeback. */
3270 ctxt->dst.type = OP_NONE;
3271 return X86EMUL_CONTINUE;
3274 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3278 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3279 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3280 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3281 return emulate_gp(ctxt, 0);
3283 return X86EMUL_CONTINUE;
3286 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3290 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3291 return emulate_gp(ctxt, 0);
3293 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3294 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3295 return X86EMUL_CONTINUE;
3298 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3300 if (ctxt->modrm_reg > VCPU_SREG_GS)
3301 return emulate_ud(ctxt);
3303 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3304 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3305 ctxt->dst.bytes = 2;
3306 return X86EMUL_CONTINUE;
3309 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3311 u16 sel = ctxt->src.val;
3313 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3314 return emulate_ud(ctxt);
3316 if (ctxt->modrm_reg == VCPU_SREG_SS)
3317 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3319 /* Disable writeback. */
3320 ctxt->dst.type = OP_NONE;
3321 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3324 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3326 u16 sel = ctxt->src.val;
3328 /* Disable writeback. */
3329 ctxt->dst.type = OP_NONE;
3330 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3333 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3335 u16 sel = ctxt->src.val;
3337 /* Disable writeback. */
3338 ctxt->dst.type = OP_NONE;
3339 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3342 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3347 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3348 if (rc == X86EMUL_CONTINUE)
3349 ctxt->ops->invlpg(ctxt, linear);
3350 /* Disable writeback. */
3351 ctxt->dst.type = OP_NONE;
3352 return X86EMUL_CONTINUE;
3355 static int em_clts(struct x86_emulate_ctxt *ctxt)
3359 cr0 = ctxt->ops->get_cr(ctxt, 0);
3361 ctxt->ops->set_cr(ctxt, 0, cr0);
3362 return X86EMUL_CONTINUE;
3365 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3367 int rc = ctxt->ops->fix_hypercall(ctxt);
3369 if (rc != X86EMUL_CONTINUE)
3372 /* Let the processor re-execute the fixed hypercall */
3373 ctxt->_eip = ctxt->eip;
3374 /* Disable writeback. */
3375 ctxt->dst.type = OP_NONE;
3376 return X86EMUL_CONTINUE;
3379 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3380 void (*get)(struct x86_emulate_ctxt *ctxt,
3381 struct desc_ptr *ptr))
3383 struct desc_ptr desc_ptr;
3385 if (ctxt->mode == X86EMUL_MODE_PROT64)
3387 get(ctxt, &desc_ptr);
3388 if (ctxt->op_bytes == 2) {
3390 desc_ptr.address &= 0x00ffffff;
3392 /* Disable writeback. */
3393 ctxt->dst.type = OP_NONE;
3394 return segmented_write(ctxt, ctxt->dst.addr.mem,
3395 &desc_ptr, 2 + ctxt->op_bytes);
3398 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3400 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3403 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3405 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3408 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3410 struct desc_ptr desc_ptr;
3413 if (ctxt->mode == X86EMUL_MODE_PROT64)
3415 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3416 &desc_ptr.size, &desc_ptr.address,
3418 if (rc != X86EMUL_CONTINUE)
3420 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3421 is_noncanonical_address(desc_ptr.address))
3422 return emulate_gp(ctxt, 0);
3424 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3426 ctxt->ops->set_idt(ctxt, &desc_ptr);
3427 /* Disable writeback. */
3428 ctxt->dst.type = OP_NONE;
3429 return X86EMUL_CONTINUE;
3432 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3434 return em_lgdt_lidt(ctxt, true);
3437 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3439 return em_lgdt_lidt(ctxt, false);
3442 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3444 if (ctxt->dst.type == OP_MEM)
3445 ctxt->dst.bytes = 2;
3446 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3447 return X86EMUL_CONTINUE;
3450 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3452 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3453 | (ctxt->src.val & 0x0f));
3454 ctxt->dst.type = OP_NONE;
3455 return X86EMUL_CONTINUE;
3458 static int em_loop(struct x86_emulate_ctxt *ctxt)
3460 int rc = X86EMUL_CONTINUE;
3462 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3463 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3464 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3465 rc = jmp_rel(ctxt, ctxt->src.val);
3470 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3472 int rc = X86EMUL_CONTINUE;
3474 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3475 rc = jmp_rel(ctxt, ctxt->src.val);
3480 static int em_in(struct x86_emulate_ctxt *ctxt)
3482 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3484 return X86EMUL_IO_NEEDED;
3486 return X86EMUL_CONTINUE;
3489 static int em_out(struct x86_emulate_ctxt *ctxt)
3491 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3493 /* Disable writeback. */
3494 ctxt->dst.type = OP_NONE;
3495 return X86EMUL_CONTINUE;
3498 static int em_cli(struct x86_emulate_ctxt *ctxt)
3500 if (emulator_bad_iopl(ctxt))
3501 return emulate_gp(ctxt, 0);
3503 ctxt->eflags &= ~X86_EFLAGS_IF;
3504 return X86EMUL_CONTINUE;
3507 static int em_sti(struct x86_emulate_ctxt *ctxt)
3509 if (emulator_bad_iopl(ctxt))
3510 return emulate_gp(ctxt, 0);
3512 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3513 ctxt->eflags |= X86_EFLAGS_IF;
3514 return X86EMUL_CONTINUE;
3517 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3519 u32 eax, ebx, ecx, edx;
3521 eax = reg_read(ctxt, VCPU_REGS_RAX);
3522 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3523 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3524 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3525 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3526 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3527 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3528 return X86EMUL_CONTINUE;
3531 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3535 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3537 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3539 ctxt->eflags &= ~0xffUL;
3540 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3541 return X86EMUL_CONTINUE;
3544 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3546 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3547 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3548 return X86EMUL_CONTINUE;
3551 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3553 switch (ctxt->op_bytes) {
3554 #ifdef CONFIG_X86_64
3556 asm("bswap %0" : "+r"(ctxt->dst.val));
3560 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3563 return X86EMUL_CONTINUE;
3566 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3568 /* emulating clflush regardless of cpuid */
3569 return X86EMUL_CONTINUE;
3572 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3574 ctxt->dst.val = (s32) ctxt->src.val;
3575 return X86EMUL_CONTINUE;
3578 static bool valid_cr(int nr)
3590 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3592 if (!valid_cr(ctxt->modrm_reg))
3593 return emulate_ud(ctxt);
3595 return X86EMUL_CONTINUE;
3598 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3600 u64 new_val = ctxt->src.val64;
3601 int cr = ctxt->modrm_reg;
3604 static u64 cr_reserved_bits[] = {
3605 0xffffffff00000000ULL,
3606 0, 0, 0, /* CR3 checked later */
3613 return emulate_ud(ctxt);
3615 if (new_val & cr_reserved_bits[cr])
3616 return emulate_gp(ctxt, 0);
3621 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3622 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3623 return emulate_gp(ctxt, 0);
3625 cr4 = ctxt->ops->get_cr(ctxt, 4);
3626 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3628 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3629 !(cr4 & X86_CR4_PAE))
3630 return emulate_gp(ctxt, 0);
3637 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3638 if (efer & EFER_LMA)
3639 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3642 return emulate_gp(ctxt, 0);
3647 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3649 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3650 return emulate_gp(ctxt, 0);
3656 return X86EMUL_CONTINUE;
3659 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3663 ctxt->ops->get_dr(ctxt, 7, &dr7);
3665 /* Check if DR7.Global_Enable is set */
3666 return dr7 & (1 << 13);
3669 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3671 int dr = ctxt->modrm_reg;
3675 return emulate_ud(ctxt);
3677 cr4 = ctxt->ops->get_cr(ctxt, 4);
3678 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3679 return emulate_ud(ctxt);
3681 if (check_dr7_gd(ctxt)) {
3684 ctxt->ops->get_dr(ctxt, 6, &dr6);
3686 dr6 |= DR6_BD | DR6_RTM;
3687 ctxt->ops->set_dr(ctxt, 6, dr6);
3688 return emulate_db(ctxt);
3691 return X86EMUL_CONTINUE;
3694 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3696 u64 new_val = ctxt->src.val64;
3697 int dr = ctxt->modrm_reg;
3699 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3700 return emulate_gp(ctxt, 0);
3702 return check_dr_read(ctxt);
3705 static int check_svme(struct x86_emulate_ctxt *ctxt)
3709 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3711 if (!(efer & EFER_SVME))
3712 return emulate_ud(ctxt);
3714 return X86EMUL_CONTINUE;
3717 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3719 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3721 /* Valid physical address? */
3722 if (rax & 0xffff000000000000ULL)
3723 return emulate_gp(ctxt, 0);
3725 return check_svme(ctxt);
3728 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3730 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3732 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3733 return emulate_ud(ctxt);
3735 return X86EMUL_CONTINUE;
3738 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3740 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3741 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3743 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3744 ctxt->ops->check_pmc(ctxt, rcx))
3745 return emulate_gp(ctxt, 0);
3747 return X86EMUL_CONTINUE;
3750 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3752 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3753 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3754 return emulate_gp(ctxt, 0);
3756 return X86EMUL_CONTINUE;
3759 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3761 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3762 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3763 return emulate_gp(ctxt, 0);
3765 return X86EMUL_CONTINUE;
3768 #define D(_y) { .flags = (_y) }
3769 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3770 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3771 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3772 #define N D(NotImpl)
3773 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3774 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3775 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3776 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3777 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3778 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3779 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3780 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3781 #define II(_f, _e, _i) \
3782 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3783 #define IIP(_f, _e, _i, _p) \
3784 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3785 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3786 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3788 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3789 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3790 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3791 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3792 #define I2bvIP(_f, _e, _i, _p) \
3793 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3795 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3796 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3797 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3799 static const struct opcode group7_rm0[] = {
3801 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
3805 static const struct opcode group7_rm1[] = {
3806 DI(SrcNone | Priv, monitor),
3807 DI(SrcNone | Priv, mwait),
3811 static const struct opcode group7_rm3[] = {
3812 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3813 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
3814 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3815 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3816 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3817 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3818 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3819 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3822 static const struct opcode group7_rm7[] = {
3824 DIP(SrcNone, rdtscp, check_rdtsc),
3828 static const struct opcode group1[] = {
3830 F(Lock | PageTable, em_or),
3833 F(Lock | PageTable, em_and),
3839 static const struct opcode group1A[] = {
3840 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3843 static const struct opcode group2[] = {
3844 F(DstMem | ModRM, em_rol),
3845 F(DstMem | ModRM, em_ror),
3846 F(DstMem | ModRM, em_rcl),
3847 F(DstMem | ModRM, em_rcr),
3848 F(DstMem | ModRM, em_shl),
3849 F(DstMem | ModRM, em_shr),
3850 F(DstMem | ModRM, em_shl),
3851 F(DstMem | ModRM, em_sar),
3854 static const struct opcode group3[] = {
3855 F(DstMem | SrcImm | NoWrite, em_test),
3856 F(DstMem | SrcImm | NoWrite, em_test),
3857 F(DstMem | SrcNone | Lock, em_not),
3858 F(DstMem | SrcNone | Lock, em_neg),
3859 F(DstXacc | Src2Mem, em_mul_ex),
3860 F(DstXacc | Src2Mem, em_imul_ex),
3861 F(DstXacc | Src2Mem, em_div_ex),
3862 F(DstXacc | Src2Mem, em_idiv_ex),
3865 static const struct opcode group4[] = {
3866 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3867 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3871 static const struct opcode group5[] = {
3872 F(DstMem | SrcNone | Lock, em_inc),
3873 F(DstMem | SrcNone | Lock, em_dec),
3874 I(SrcMem | NearBranch, em_call_near_abs),
3875 I(SrcMemFAddr | ImplicitOps, em_call_far),
3876 I(SrcMem | NearBranch, em_jmp_abs),
3877 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3878 I(SrcMem | Stack, em_push), D(Undefined),
3881 static const struct opcode group6[] = {
3882 DI(Prot | DstMem, sldt),
3883 DI(Prot | DstMem, str),
3884 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3885 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3889 static const struct group_dual group7 = { {
3890 II(Mov | DstMem, em_sgdt, sgdt),
3891 II(Mov | DstMem, em_sidt, sidt),
3892 II(SrcMem | Priv, em_lgdt, lgdt),
3893 II(SrcMem | Priv, em_lidt, lidt),
3894 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3895 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3896 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3900 N, EXT(0, group7_rm3),
3901 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3902 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3906 static const struct opcode group8[] = {
3908 F(DstMem | SrcImmByte | NoWrite, em_bt),
3909 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3910 F(DstMem | SrcImmByte | Lock, em_btr),
3911 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3914 static const struct group_dual group9 = { {
3915 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3917 N, N, N, N, N, N, N, N,
3920 static const struct opcode group11[] = {
3921 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3925 static const struct gprefix pfx_0f_ae_7 = {
3926 I(SrcMem | ByteOp, em_clflush), N, N, N,
3929 static const struct group_dual group15 = { {
3930 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3932 N, N, N, N, N, N, N, N,
3935 static const struct gprefix pfx_0f_6f_0f_7f = {
3936 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3939 static const struct instr_dual instr_dual_0f_2b = {
3943 static const struct gprefix pfx_0f_2b = {
3944 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3947 static const struct gprefix pfx_0f_28_0f_29 = {
3948 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3951 static const struct gprefix pfx_0f_e7 = {
3952 N, I(Sse, em_mov), N, N,
3955 static const struct escape escape_d9 = { {
3956 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3959 N, N, N, N, N, N, N, N,
3961 N, N, N, N, N, N, N, N,
3963 N, N, N, N, N, N, N, N,
3965 N, N, N, N, N, N, N, N,
3967 N, N, N, N, N, N, N, N,
3969 N, N, N, N, N, N, N, N,
3971 N, N, N, N, N, N, N, N,
3973 N, N, N, N, N, N, N, N,
3976 static const struct escape escape_db = { {
3977 N, N, N, N, N, N, N, N,
3980 N, N, N, N, N, N, N, N,
3982 N, N, N, N, N, N, N, N,
3984 N, N, N, N, N, N, N, N,
3986 N, N, N, N, N, N, N, N,
3988 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3990 N, N, N, N, N, N, N, N,
3992 N, N, N, N, N, N, N, N,
3994 N, N, N, N, N, N, N, N,
3997 static const struct escape escape_dd = { {
3998 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4001 N, N, N, N, N, N, N, N,
4003 N, N, N, N, N, N, N, N,
4005 N, N, N, N, N, N, N, N,
4007 N, N, N, N, N, N, N, N,
4009 N, N, N, N, N, N, N, N,
4011 N, N, N, N, N, N, N, N,
4013 N, N, N, N, N, N, N, N,
4015 N, N, N, N, N, N, N, N,
4018 static const struct instr_dual instr_dual_0f_c3 = {
4019 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4022 static const struct mode_dual mode_dual_63 = {
4023 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4026 static const struct opcode opcode_table[256] = {
4028 F6ALU(Lock, em_add),
4029 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4030 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4032 F6ALU(Lock | PageTable, em_or),
4033 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4036 F6ALU(Lock, em_adc),
4037 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4038 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4040 F6ALU(Lock, em_sbb),
4041 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4042 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4044 F6ALU(Lock | PageTable, em_and), N, N,
4046 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4048 F6ALU(Lock, em_xor), N, N,
4050 F6ALU(NoWrite, em_cmp), N, N,
4052 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4054 X8(I(SrcReg | Stack, em_push)),
4056 X8(I(DstReg | Stack, em_pop)),
4058 I(ImplicitOps | Stack | No64, em_pusha),
4059 I(ImplicitOps | Stack | No64, em_popa),
4060 N, MD(ModRM, &mode_dual_63),
4063 I(SrcImm | Mov | Stack, em_push),
4064 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4065 I(SrcImmByte | Mov | Stack, em_push),
4066 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4067 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4068 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4070 X16(D(SrcImmByte | NearBranch)),
4072 G(ByteOp | DstMem | SrcImm, group1),
4073 G(DstMem | SrcImm, group1),
4074 G(ByteOp | DstMem | SrcImm | No64, group1),
4075 G(DstMem | SrcImmByte, group1),
4076 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4077 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4079 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4080 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4081 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4082 D(ModRM | SrcMem | NoAccess | DstReg),
4083 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4086 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4088 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4089 I(SrcImmFAddr | No64, em_call_far), N,
4090 II(ImplicitOps | Stack, em_pushf, pushf),
4091 II(ImplicitOps | Stack, em_popf, popf),
4092 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4094 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4095 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4096 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4097 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4099 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4100 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4101 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4102 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4104 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4106 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4108 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4109 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4110 I(ImplicitOps | NearBranch, em_ret),
4111 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4112 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4113 G(ByteOp, group11), G(0, group11),
4115 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4116 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4117 I(ImplicitOps, em_ret_far),
4118 D(ImplicitOps), DI(SrcImmByte, intn),
4119 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4121 G(Src2One | ByteOp, group2), G(Src2One, group2),
4122 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4123 I(DstAcc | SrcImmUByte | No64, em_aam),
4124 I(DstAcc | SrcImmUByte | No64, em_aad),
4125 F(DstAcc | ByteOp | No64, em_salc),
4126 I(DstAcc | SrcXLat | ByteOp, em_mov),
4128 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4130 X3(I(SrcImmByte | NearBranch, em_loop)),
4131 I(SrcImmByte | NearBranch, em_jcxz),
4132 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4133 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4135 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4136 I(SrcImmFAddr | No64, em_jmp_far),
4137 D(SrcImmByte | ImplicitOps | NearBranch),
4138 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4139 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4141 N, DI(ImplicitOps, icebp), N, N,
4142 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4143 G(ByteOp, group3), G(0, group3),
4145 D(ImplicitOps), D(ImplicitOps),
4146 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4147 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4150 static const struct opcode twobyte_table[256] = {
4152 G(0, group6), GD(0, &group7), N, N,
4153 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4154 II(ImplicitOps | Priv, em_clts, clts), N,
4155 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4156 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4158 N, N, N, N, N, N, N, N,
4159 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4160 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4162 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4163 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4164 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4166 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4169 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4170 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4171 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4174 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4175 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4176 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4177 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4178 I(ImplicitOps | EmulateOnUD, em_sysenter),
4179 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4181 N, N, N, N, N, N, N, N,
4183 X16(D(DstReg | SrcMem | ModRM)),
4185 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4190 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4195 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4197 X16(D(SrcImm | NearBranch)),
4199 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4201 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4202 II(ImplicitOps, em_cpuid, cpuid),
4203 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4204 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4205 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4207 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4208 II(No64 | EmulateOnUD | ImplicitOps, em_rsm, rsm),
4209 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4210 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4211 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4212 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4214 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4215 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4216 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4217 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4218 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4219 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4223 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4224 I(DstReg | SrcMem | ModRM, em_bsf_c),
4225 I(DstReg | SrcMem | ModRM, em_bsr_c),
4226 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4228 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4229 N, ID(0, &instr_dual_0f_c3),
4230 N, N, N, GD(0, &group9),
4232 X8(I(DstReg, em_bswap)),
4234 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4236 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4237 N, N, N, N, N, N, N, N,
4239 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4242 static const struct instr_dual instr_dual_0f_38_f0 = {
4243 I(DstReg | SrcMem | Mov, em_movbe), N
4246 static const struct instr_dual instr_dual_0f_38_f1 = {
4247 I(DstMem | SrcReg | Mov, em_movbe), N
4250 static const struct gprefix three_byte_0f_38_f0 = {
4251 ID(0, &instr_dual_0f_38_f0), N, N, N
4254 static const struct gprefix three_byte_0f_38_f1 = {
4255 ID(0, &instr_dual_0f_38_f1), N, N, N
4259 * Insns below are selected by the prefix which indexed by the third opcode
4262 static const struct opcode opcode_map_0f_38[256] = {
4264 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4266 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4268 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4269 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4290 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4294 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4300 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4301 unsigned size, bool sign_extension)
4303 int rc = X86EMUL_CONTINUE;
4307 op->addr.mem.ea = ctxt->_eip;
4308 /* NB. Immediates are sign-extended as necessary. */
4309 switch (op->bytes) {
4311 op->val = insn_fetch(s8, ctxt);
4314 op->val = insn_fetch(s16, ctxt);
4317 op->val = insn_fetch(s32, ctxt);
4320 op->val = insn_fetch(s64, ctxt);
4323 if (!sign_extension) {
4324 switch (op->bytes) {
4332 op->val &= 0xffffffff;
4340 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4343 int rc = X86EMUL_CONTINUE;
4347 decode_register_operand(ctxt, op);
4350 rc = decode_imm(ctxt, op, 1, false);
4353 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4357 if (ctxt->d & BitOp)
4358 fetch_bit_operand(ctxt);
4359 op->orig_val = op->val;
4362 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4366 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4367 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4368 fetch_register_operand(op);
4369 op->orig_val = op->val;
4373 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4374 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4375 fetch_register_operand(op);
4376 op->orig_val = op->val;
4379 if (ctxt->d & ByteOp) {
4384 op->bytes = ctxt->op_bytes;
4385 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4386 fetch_register_operand(op);
4387 op->orig_val = op->val;
4391 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4393 register_address(ctxt, VCPU_REGS_RDI);
4394 op->addr.mem.seg = VCPU_SREG_ES;
4401 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4402 fetch_register_operand(op);
4407 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4410 rc = decode_imm(ctxt, op, 1, true);
4418 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4421 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4424 ctxt->memop.bytes = 1;
4425 if (ctxt->memop.type == OP_REG) {
4426 ctxt->memop.addr.reg = decode_register(ctxt,
4427 ctxt->modrm_rm, true);
4428 fetch_register_operand(&ctxt->memop);
4432 ctxt->memop.bytes = 2;
4435 ctxt->memop.bytes = 4;
4438 rc = decode_imm(ctxt, op, 2, false);
4441 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4445 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4447 register_address(ctxt, VCPU_REGS_RSI);
4448 op->addr.mem.seg = ctxt->seg_override;
4454 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4457 reg_read(ctxt, VCPU_REGS_RBX) +
4458 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4459 op->addr.mem.seg = ctxt->seg_override;
4464 op->addr.mem.ea = ctxt->_eip;
4465 op->bytes = ctxt->op_bytes + 2;
4466 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4469 ctxt->memop.bytes = ctxt->op_bytes + 2;
4473 op->val = VCPU_SREG_ES;
4477 op->val = VCPU_SREG_CS;
4481 op->val = VCPU_SREG_SS;
4485 op->val = VCPU_SREG_DS;
4489 op->val = VCPU_SREG_FS;
4493 op->val = VCPU_SREG_GS;
4496 /* Special instructions do their own operand decoding. */
4498 op->type = OP_NONE; /* Disable writeback. */
4506 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4508 int rc = X86EMUL_CONTINUE;
4509 int mode = ctxt->mode;
4510 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4511 bool op_prefix = false;
4512 bool has_seg_override = false;
4513 struct opcode opcode;
4515 ctxt->memop.type = OP_NONE;
4516 ctxt->memopp = NULL;
4517 ctxt->_eip = ctxt->eip;
4518 ctxt->fetch.ptr = ctxt->fetch.data;
4519 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4520 ctxt->opcode_len = 1;
4522 memcpy(ctxt->fetch.data, insn, insn_len);
4524 rc = __do_insn_fetch_bytes(ctxt, 1);
4525 if (rc != X86EMUL_CONTINUE)
4530 case X86EMUL_MODE_REAL:
4531 case X86EMUL_MODE_VM86:
4532 case X86EMUL_MODE_PROT16:
4533 def_op_bytes = def_ad_bytes = 2;
4535 case X86EMUL_MODE_PROT32:
4536 def_op_bytes = def_ad_bytes = 4;
4538 #ifdef CONFIG_X86_64
4539 case X86EMUL_MODE_PROT64:
4545 return EMULATION_FAILED;
4548 ctxt->op_bytes = def_op_bytes;
4549 ctxt->ad_bytes = def_ad_bytes;
4551 /* Legacy prefixes. */
4553 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4554 case 0x66: /* operand-size override */
4556 /* switch between 2/4 bytes */
4557 ctxt->op_bytes = def_op_bytes ^ 6;
4559 case 0x67: /* address-size override */
4560 if (mode == X86EMUL_MODE_PROT64)
4561 /* switch between 4/8 bytes */
4562 ctxt->ad_bytes = def_ad_bytes ^ 12;
4564 /* switch between 2/4 bytes */
4565 ctxt->ad_bytes = def_ad_bytes ^ 6;
4567 case 0x26: /* ES override */
4568 case 0x2e: /* CS override */
4569 case 0x36: /* SS override */
4570 case 0x3e: /* DS override */
4571 has_seg_override = true;
4572 ctxt->seg_override = (ctxt->b >> 3) & 3;
4574 case 0x64: /* FS override */
4575 case 0x65: /* GS override */
4576 has_seg_override = true;
4577 ctxt->seg_override = ctxt->b & 7;
4579 case 0x40 ... 0x4f: /* REX */
4580 if (mode != X86EMUL_MODE_PROT64)
4582 ctxt->rex_prefix = ctxt->b;
4584 case 0xf0: /* LOCK */
4585 ctxt->lock_prefix = 1;
4587 case 0xf2: /* REPNE/REPNZ */
4588 case 0xf3: /* REP/REPE/REPZ */
4589 ctxt->rep_prefix = ctxt->b;
4595 /* Any legacy prefix after a REX prefix nullifies its effect. */
4597 ctxt->rex_prefix = 0;
4603 if (ctxt->rex_prefix & 8)
4604 ctxt->op_bytes = 8; /* REX.W */
4606 /* Opcode byte(s). */
4607 opcode = opcode_table[ctxt->b];
4608 /* Two-byte opcode? */
4609 if (ctxt->b == 0x0f) {
4610 ctxt->opcode_len = 2;
4611 ctxt->b = insn_fetch(u8, ctxt);
4612 opcode = twobyte_table[ctxt->b];
4614 /* 0F_38 opcode map */
4615 if (ctxt->b == 0x38) {
4616 ctxt->opcode_len = 3;
4617 ctxt->b = insn_fetch(u8, ctxt);
4618 opcode = opcode_map_0f_38[ctxt->b];
4621 ctxt->d = opcode.flags;
4623 if (ctxt->d & ModRM)
4624 ctxt->modrm = insn_fetch(u8, ctxt);
4626 /* vex-prefix instructions are not implemented */
4627 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4628 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4632 while (ctxt->d & GroupMask) {
4633 switch (ctxt->d & GroupMask) {
4635 goffset = (ctxt->modrm >> 3) & 7;
4636 opcode = opcode.u.group[goffset];
4639 goffset = (ctxt->modrm >> 3) & 7;
4640 if ((ctxt->modrm >> 6) == 3)
4641 opcode = opcode.u.gdual->mod3[goffset];
4643 opcode = opcode.u.gdual->mod012[goffset];
4646 goffset = ctxt->modrm & 7;
4647 opcode = opcode.u.group[goffset];
4650 if (ctxt->rep_prefix && op_prefix)
4651 return EMULATION_FAILED;
4652 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4653 switch (simd_prefix) {
4654 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4655 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4656 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4657 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4661 if (ctxt->modrm > 0xbf)
4662 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4664 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4667 if ((ctxt->modrm >> 6) == 3)
4668 opcode = opcode.u.idual->mod3;
4670 opcode = opcode.u.idual->mod012;
4673 if (ctxt->mode == X86EMUL_MODE_PROT64)
4674 opcode = opcode.u.mdual->mode64;
4676 opcode = opcode.u.mdual->mode32;
4679 return EMULATION_FAILED;
4682 ctxt->d &= ~(u64)GroupMask;
4683 ctxt->d |= opcode.flags;
4688 return EMULATION_FAILED;
4690 ctxt->execute = opcode.u.execute;
4692 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4693 return EMULATION_FAILED;
4695 if (unlikely(ctxt->d &
4696 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4699 * These are copied unconditionally here, and checked unconditionally
4700 * in x86_emulate_insn.
4702 ctxt->check_perm = opcode.check_perm;
4703 ctxt->intercept = opcode.intercept;
4705 if (ctxt->d & NotImpl)
4706 return EMULATION_FAILED;
4708 if (mode == X86EMUL_MODE_PROT64) {
4709 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4711 else if (ctxt->d & NearBranch)
4715 if (ctxt->d & Op3264) {
4716 if (mode == X86EMUL_MODE_PROT64)
4722 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4726 ctxt->op_bytes = 16;
4727 else if (ctxt->d & Mmx)
4731 /* ModRM and SIB bytes. */
4732 if (ctxt->d & ModRM) {
4733 rc = decode_modrm(ctxt, &ctxt->memop);
4734 if (!has_seg_override) {
4735 has_seg_override = true;
4736 ctxt->seg_override = ctxt->modrm_seg;
4738 } else if (ctxt->d & MemAbs)
4739 rc = decode_abs(ctxt, &ctxt->memop);
4740 if (rc != X86EMUL_CONTINUE)
4743 if (!has_seg_override)
4744 ctxt->seg_override = VCPU_SREG_DS;
4746 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4749 * Decode and fetch the source operand: register, memory
4752 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4753 if (rc != X86EMUL_CONTINUE)
4757 * Decode and fetch the second source operand: register, memory
4760 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4761 if (rc != X86EMUL_CONTINUE)
4764 /* Decode and fetch the destination operand: register or memory. */
4765 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4767 if (ctxt->rip_relative)
4768 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4769 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4772 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4775 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4777 return ctxt->d & PageTable;
4780 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4782 /* The second termination condition only applies for REPE
4783 * and REPNE. Test if the repeat string operation prefix is
4784 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4785 * corresponding termination condition according to:
4786 * - if REPE/REPZ and ZF = 0 then done
4787 * - if REPNE/REPNZ and ZF = 1 then done
4789 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4790 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4791 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4792 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
4793 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4794 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
4800 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4804 ctxt->ops->get_fpu(ctxt);
4805 asm volatile("1: fwait \n\t"
4807 ".pushsection .fixup,\"ax\" \n\t"
4809 "movb $1, %[fault] \n\t"
4812 _ASM_EXTABLE(1b, 3b)
4813 : [fault]"+qm"(fault));
4814 ctxt->ops->put_fpu(ctxt);
4816 if (unlikely(fault))
4817 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4819 return X86EMUL_CONTINUE;
4822 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4825 if (op->type == OP_MM)
4826 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4829 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4831 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4832 if (!(ctxt->d & ByteOp))
4833 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4834 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4835 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4837 : "c"(ctxt->src2.val));
4838 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4839 if (!fop) /* exception is returned in fop variable */
4840 return emulate_de(ctxt);
4841 return X86EMUL_CONTINUE;
4844 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4846 memset(&ctxt->rip_relative, 0,
4847 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4849 ctxt->io_read.pos = 0;
4850 ctxt->io_read.end = 0;
4851 ctxt->mem_read.end = 0;
4854 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4856 const struct x86_emulate_ops *ops = ctxt->ops;
4857 int rc = X86EMUL_CONTINUE;
4858 int saved_dst_type = ctxt->dst.type;
4860 ctxt->mem_read.pos = 0;
4862 /* LOCK prefix is allowed only with some instructions */
4863 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4864 rc = emulate_ud(ctxt);
4868 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4869 rc = emulate_ud(ctxt);
4873 if (unlikely(ctxt->d &
4874 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4875 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4876 (ctxt->d & Undefined)) {
4877 rc = emulate_ud(ctxt);
4881 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4882 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4883 rc = emulate_ud(ctxt);
4887 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4888 rc = emulate_nm(ctxt);
4892 if (ctxt->d & Mmx) {
4893 rc = flush_pending_x87_faults(ctxt);
4894 if (rc != X86EMUL_CONTINUE)
4897 * Now that we know the fpu is exception safe, we can fetch
4900 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4901 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4902 if (!(ctxt->d & Mov))
4903 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4906 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
4907 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4908 X86_ICPT_PRE_EXCEPT);
4909 if (rc != X86EMUL_CONTINUE)
4913 /* Instruction can only be executed in protected mode */
4914 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4915 rc = emulate_ud(ctxt);
4919 /* Privileged instruction can be executed only in CPL=0 */
4920 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4921 if (ctxt->d & PrivUD)
4922 rc = emulate_ud(ctxt);
4924 rc = emulate_gp(ctxt, 0);
4928 /* Do instruction specific permission checks */
4929 if (ctxt->d & CheckPerm) {
4930 rc = ctxt->check_perm(ctxt);
4931 if (rc != X86EMUL_CONTINUE)
4935 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
4936 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4937 X86_ICPT_POST_EXCEPT);
4938 if (rc != X86EMUL_CONTINUE)
4942 if (ctxt->rep_prefix && (ctxt->d & String)) {
4943 /* All REP prefixes have the same first termination condition */
4944 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4945 string_registers_quirk(ctxt);
4946 ctxt->eip = ctxt->_eip;
4947 ctxt->eflags &= ~X86_EFLAGS_RF;
4953 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4954 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4955 ctxt->src.valptr, ctxt->src.bytes);
4956 if (rc != X86EMUL_CONTINUE)
4958 ctxt->src.orig_val64 = ctxt->src.val64;
4961 if (ctxt->src2.type == OP_MEM) {
4962 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4963 &ctxt->src2.val, ctxt->src2.bytes);
4964 if (rc != X86EMUL_CONTINUE)
4968 if ((ctxt->d & DstMask) == ImplicitOps)
4972 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4973 /* optimisation - avoid slow emulated read if Mov */
4974 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4975 &ctxt->dst.val, ctxt->dst.bytes);
4976 if (rc != X86EMUL_CONTINUE) {
4977 if (!(ctxt->d & NoWrite) &&
4978 rc == X86EMUL_PROPAGATE_FAULT &&
4979 ctxt->exception.vector == PF_VECTOR)
4980 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4984 /* Copy full 64-bit value for CMPXCHG8B. */
4985 ctxt->dst.orig_val64 = ctxt->dst.val64;
4989 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
4990 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4991 X86_ICPT_POST_MEMACCESS);
4992 if (rc != X86EMUL_CONTINUE)
4996 if (ctxt->rep_prefix && (ctxt->d & String))
4997 ctxt->eflags |= X86_EFLAGS_RF;
4999 ctxt->eflags &= ~X86_EFLAGS_RF;
5001 if (ctxt->execute) {
5002 if (ctxt->d & Fastop) {
5003 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5004 rc = fastop(ctxt, fop);
5005 if (rc != X86EMUL_CONTINUE)
5009 rc = ctxt->execute(ctxt);
5010 if (rc != X86EMUL_CONTINUE)
5015 if (ctxt->opcode_len == 2)
5017 else if (ctxt->opcode_len == 3)
5018 goto threebyte_insn;
5021 case 0x70 ... 0x7f: /* jcc (short) */
5022 if (test_cc(ctxt->b, ctxt->eflags))
5023 rc = jmp_rel(ctxt, ctxt->src.val);
5025 case 0x8d: /* lea r16/r32, m */
5026 ctxt->dst.val = ctxt->src.addr.mem.ea;
5028 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5029 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5030 ctxt->dst.type = OP_NONE;
5034 case 0x98: /* cbw/cwde/cdqe */
5035 switch (ctxt->op_bytes) {
5036 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5037 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5038 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5041 case 0xcc: /* int3 */
5042 rc = emulate_int(ctxt, 3);
5044 case 0xcd: /* int n */
5045 rc = emulate_int(ctxt, ctxt->src.val);
5047 case 0xce: /* into */
5048 if (ctxt->eflags & X86_EFLAGS_OF)
5049 rc = emulate_int(ctxt, 4);
5051 case 0xe9: /* jmp rel */
5052 case 0xeb: /* jmp rel short */
5053 rc = jmp_rel(ctxt, ctxt->src.val);
5054 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5056 case 0xf4: /* hlt */
5057 ctxt->ops->halt(ctxt);
5059 case 0xf5: /* cmc */
5060 /* complement carry flag from eflags reg */
5061 ctxt->eflags ^= X86_EFLAGS_CF;
5063 case 0xf8: /* clc */
5064 ctxt->eflags &= ~X86_EFLAGS_CF;
5066 case 0xf9: /* stc */
5067 ctxt->eflags |= X86_EFLAGS_CF;
5069 case 0xfc: /* cld */
5070 ctxt->eflags &= ~X86_EFLAGS_DF;
5072 case 0xfd: /* std */
5073 ctxt->eflags |= X86_EFLAGS_DF;
5076 goto cannot_emulate;
5079 if (rc != X86EMUL_CONTINUE)
5083 if (ctxt->d & SrcWrite) {
5084 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5085 rc = writeback(ctxt, &ctxt->src);
5086 if (rc != X86EMUL_CONTINUE)
5089 if (!(ctxt->d & NoWrite)) {
5090 rc = writeback(ctxt, &ctxt->dst);
5091 if (rc != X86EMUL_CONTINUE)
5096 * restore dst type in case the decoding will be reused
5097 * (happens for string instruction )
5099 ctxt->dst.type = saved_dst_type;
5101 if ((ctxt->d & SrcMask) == SrcSI)
5102 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5104 if ((ctxt->d & DstMask) == DstDI)
5105 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5107 if (ctxt->rep_prefix && (ctxt->d & String)) {
5109 struct read_cache *r = &ctxt->io_read;
5110 if ((ctxt->d & SrcMask) == SrcSI)
5111 count = ctxt->src.count;
5113 count = ctxt->dst.count;
5114 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5116 if (!string_insn_completed(ctxt)) {
5118 * Re-enter guest when pio read ahead buffer is empty
5119 * or, if it is not used, after each 1024 iteration.
5121 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5122 (r->end == 0 || r->end != r->pos)) {
5124 * Reset read cache. Usually happens before
5125 * decode, but since instruction is restarted
5126 * we have to do it here.
5128 ctxt->mem_read.end = 0;
5129 writeback_registers(ctxt);
5130 return EMULATION_RESTART;
5132 goto done; /* skip rip writeback */
5134 ctxt->eflags &= ~X86_EFLAGS_RF;
5137 ctxt->eip = ctxt->_eip;
5140 if (rc == X86EMUL_PROPAGATE_FAULT) {
5141 WARN_ON(ctxt->exception.vector > 0x1f);
5142 ctxt->have_exception = true;
5144 if (rc == X86EMUL_INTERCEPTED)
5145 return EMULATION_INTERCEPTED;
5147 if (rc == X86EMUL_CONTINUE)
5148 writeback_registers(ctxt);
5150 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5154 case 0x09: /* wbinvd */
5155 (ctxt->ops->wbinvd)(ctxt);
5157 case 0x08: /* invd */
5158 case 0x0d: /* GrpP (prefetch) */
5159 case 0x18: /* Grp16 (prefetch/nop) */
5160 case 0x1f: /* nop */
5162 case 0x20: /* mov cr, reg */
5163 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5165 case 0x21: /* mov from dr to reg */
5166 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5168 case 0x40 ... 0x4f: /* cmov */
5169 if (test_cc(ctxt->b, ctxt->eflags))
5170 ctxt->dst.val = ctxt->src.val;
5171 else if (ctxt->op_bytes != 4)
5172 ctxt->dst.type = OP_NONE; /* no writeback */
5174 case 0x80 ... 0x8f: /* jnz rel, etc*/
5175 if (test_cc(ctxt->b, ctxt->eflags))
5176 rc = jmp_rel(ctxt, ctxt->src.val);
5178 case 0x90 ... 0x9f: /* setcc r/m8 */
5179 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5181 case 0xb6 ... 0xb7: /* movzx */
5182 ctxt->dst.bytes = ctxt->op_bytes;
5183 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5184 : (u16) ctxt->src.val;
5186 case 0xbe ... 0xbf: /* movsx */
5187 ctxt->dst.bytes = ctxt->op_bytes;
5188 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5189 (s16) ctxt->src.val;
5192 goto cannot_emulate;
5197 if (rc != X86EMUL_CONTINUE)
5203 return EMULATION_FAILED;
5206 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5208 invalidate_registers(ctxt);
5211 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5213 writeback_registers(ctxt);