1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <linux/nospec.h>
27 #include <asm/kvm_emulate.h>
28 #include <linux/stringify.h>
29 #include <asm/debugreg.h>
30 #include <asm/nospec-branch.h>
39 #define OpImplicit 1ull /* No generic decode */
40 #define OpReg 2ull /* Register */
41 #define OpMem 3ull /* Memory */
42 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
43 #define OpDI 5ull /* ES:DI/EDI/RDI */
44 #define OpMem64 6ull /* Memory, 64-bit */
45 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
46 #define OpDX 8ull /* DX register */
47 #define OpCL 9ull /* CL register (for shifts) */
48 #define OpImmByte 10ull /* 8-bit sign extended immediate */
49 #define OpOne 11ull /* Implied 1 */
50 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
51 #define OpMem16 13ull /* Memory operand (16-bit). */
52 #define OpMem32 14ull /* Memory operand (32-bit). */
53 #define OpImmU 15ull /* Immediate operand, zero extended */
54 #define OpSI 16ull /* SI/ESI/RSI */
55 #define OpImmFAddr 17ull /* Immediate far address */
56 #define OpMemFAddr 18ull /* Far address in memory */
57 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
58 #define OpES 20ull /* ES */
59 #define OpCS 21ull /* CS */
60 #define OpSS 22ull /* SS */
61 #define OpDS 23ull /* DS */
62 #define OpFS 24ull /* FS */
63 #define OpGS 25ull /* GS */
64 #define OpMem8 26ull /* 8-bit zero extended memory operand */
65 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
66 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
67 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
68 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
70 #define OpBits 5 /* Width of operand field */
71 #define OpMask ((1ull << OpBits) - 1)
74 * Opcode effective-address decode tables.
75 * Note that we only emulate instructions that have at least one memory
76 * operand (excluding implicit stack references). We assume that stack
77 * references and instruction fetches will never occur in special memory
78 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
82 /* Operand sizes: 8-bit operands or specified/overridden size. */
83 #define ByteOp (1<<0) /* 8-bit operands. */
84 /* Destination operand type. */
86 #define ImplicitOps (OpImplicit << DstShift)
87 #define DstReg (OpReg << DstShift)
88 #define DstMem (OpMem << DstShift)
89 #define DstAcc (OpAcc << DstShift)
90 #define DstDI (OpDI << DstShift)
91 #define DstMem64 (OpMem64 << DstShift)
92 #define DstMem16 (OpMem16 << DstShift)
93 #define DstImmUByte (OpImmUByte << DstShift)
94 #define DstDX (OpDX << DstShift)
95 #define DstAccLo (OpAccLo << DstShift)
96 #define DstMask (OpMask << DstShift)
97 /* Source operand type. */
99 #define SrcNone (OpNone << SrcShift)
100 #define SrcReg (OpReg << SrcShift)
101 #define SrcMem (OpMem << SrcShift)
102 #define SrcMem16 (OpMem16 << SrcShift)
103 #define SrcMem32 (OpMem32 << SrcShift)
104 #define SrcImm (OpImm << SrcShift)
105 #define SrcImmByte (OpImmByte << SrcShift)
106 #define SrcOne (OpOne << SrcShift)
107 #define SrcImmUByte (OpImmUByte << SrcShift)
108 #define SrcImmU (OpImmU << SrcShift)
109 #define SrcSI (OpSI << SrcShift)
110 #define SrcXLat (OpXLat << SrcShift)
111 #define SrcImmFAddr (OpImmFAddr << SrcShift)
112 #define SrcMemFAddr (OpMemFAddr << SrcShift)
113 #define SrcAcc (OpAcc << SrcShift)
114 #define SrcImmU16 (OpImmU16 << SrcShift)
115 #define SrcImm64 (OpImm64 << SrcShift)
116 #define SrcDX (OpDX << SrcShift)
117 #define SrcMem8 (OpMem8 << SrcShift)
118 #define SrcAccHi (OpAccHi << SrcShift)
119 #define SrcMask (OpMask << SrcShift)
120 #define BitOp (1<<11)
121 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
122 #define String (1<<13) /* String instruction (rep capable) */
123 #define Stack (1<<14) /* Stack instruction (push/pop) */
124 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
125 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
126 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
127 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
128 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
129 #define Escape (5<<15) /* Escape to coprocessor instruction */
130 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
131 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
132 #define Sse (1<<18) /* SSE Vector instruction */
133 /* Generic ModRM decode. */
134 #define ModRM (1<<19)
135 /* Destination is only written; never read. */
138 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
139 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
140 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
141 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
142 #define Undefined (1<<25) /* No Such Instruction */
143 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
144 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
146 #define PageTable (1 << 29) /* instruction used to write page table */
147 #define NotImpl (1 << 30) /* instruction is not implemented */
148 /* Source 2 operand type */
149 #define Src2Shift (31)
150 #define Src2None (OpNone << Src2Shift)
151 #define Src2Mem (OpMem << Src2Shift)
152 #define Src2CL (OpCL << Src2Shift)
153 #define Src2ImmByte (OpImmByte << Src2Shift)
154 #define Src2One (OpOne << Src2Shift)
155 #define Src2Imm (OpImm << Src2Shift)
156 #define Src2ES (OpES << Src2Shift)
157 #define Src2CS (OpCS << Src2Shift)
158 #define Src2SS (OpSS << Src2Shift)
159 #define Src2DS (OpDS << Src2Shift)
160 #define Src2FS (OpFS << Src2Shift)
161 #define Src2GS (OpGS << Src2Shift)
162 #define Src2Mask (OpMask << Src2Shift)
163 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
167 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
168 #define NoWrite ((u64)1 << 45) /* No writeback */
169 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
170 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
171 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
172 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
173 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
174 #define NearBranch ((u64)1 << 52) /* Near branches */
175 #define No16 ((u64)1 << 53) /* No 16 bit operand */
176 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
177 #define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
179 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
181 #define X2(x...) x, x
182 #define X3(x...) X2(x), x
183 #define X4(x...) X2(x), X2(x)
184 #define X5(x...) X4(x), x
185 #define X6(x...) X4(x), X2(x)
186 #define X7(x...) X4(x), X3(x)
187 #define X8(x...) X4(x), X4(x)
188 #define X16(x...) X8(x), X8(x)
190 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
191 #define FASTOP_SIZE 8
194 * fastop functions have a special calling convention:
199 * flags: rflags (in/out)
200 * ex: rsi (in:fastop pointer, out:zero if exception)
202 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
203 * different operand sizes can be reached by calculation, rather than a jump
204 * table (which would be bigger than the code).
206 * fastop functions are declared as taking a never-defined fastop parameter,
207 * so they can't be called from C directly.
216 int (*execute)(struct x86_emulate_ctxt *ctxt);
217 const struct opcode *group;
218 const struct group_dual *gdual;
219 const struct gprefix *gprefix;
220 const struct escape *esc;
221 const struct instr_dual *idual;
222 const struct mode_dual *mdual;
223 void (*fastop)(struct fastop *fake);
225 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
229 struct opcode mod012[8];
230 struct opcode mod3[8];
234 struct opcode pfx_no;
235 struct opcode pfx_66;
236 struct opcode pfx_f2;
237 struct opcode pfx_f3;
242 struct opcode high[64];
246 struct opcode mod012;
251 struct opcode mode32;
252 struct opcode mode64;
255 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
257 enum x86_transfer_type {
259 X86_TRANSFER_CALL_JMP,
261 X86_TRANSFER_TASK_SWITCH,
264 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
266 if (!(ctxt->regs_valid & (1 << nr))) {
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
270 return ctxt->_regs[nr];
273 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 ctxt->regs_valid |= 1 << nr;
276 ctxt->regs_dirty |= 1 << nr;
277 return &ctxt->_regs[nr];
280 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
283 return reg_write(ctxt, nr);
286 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
290 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
291 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
294 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
296 ctxt->regs_dirty = 0;
297 ctxt->regs_valid = 0;
301 * These EFLAGS bits are restored from saved value during emulation, and
302 * any changes are written back to the saved value after emulation.
304 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
305 X86_EFLAGS_PF|X86_EFLAGS_CF)
313 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
315 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
316 #define FOP_RET "ret \n\t"
318 #define FOP_START(op) \
319 extern void em_##op(struct fastop *fake); \
320 asm(".pushsection .text, \"ax\" \n\t" \
321 ".global em_" #op " \n\t" \
328 #define FOPNOP() FOP_ALIGN FOP_RET
330 #define FOP1E(op, dst) \
331 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
333 #define FOP1EEX(op, dst) \
334 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
336 #define FASTOP1(op) \
341 ON64(FOP1E(op##q, rax)) \
344 /* 1-operand, using src2 (for MUL/DIV r/m) */
345 #define FASTOP1SRC2(op, name) \
350 ON64(FOP1E(op, rcx)) \
353 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
354 #define FASTOP1SRC2EX(op, name) \
359 ON64(FOP1EEX(op, rcx)) \
362 #define FOP2E(op, dst, src) \
363 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
365 #define FASTOP2(op) \
367 FOP2E(op##b, al, dl) \
368 FOP2E(op##w, ax, dx) \
369 FOP2E(op##l, eax, edx) \
370 ON64(FOP2E(op##q, rax, rdx)) \
373 /* 2 operand, word only */
374 #define FASTOP2W(op) \
377 FOP2E(op##w, ax, dx) \
378 FOP2E(op##l, eax, edx) \
379 ON64(FOP2E(op##q, rax, rdx)) \
382 /* 2 operand, src is CL */
383 #define FASTOP2CL(op) \
385 FOP2E(op##b, al, cl) \
386 FOP2E(op##w, ax, cl) \
387 FOP2E(op##l, eax, cl) \
388 ON64(FOP2E(op##q, rax, cl)) \
391 /* 2 operand, src and dest are reversed */
392 #define FASTOP2R(op, name) \
394 FOP2E(op##b, dl, al) \
395 FOP2E(op##w, dx, ax) \
396 FOP2E(op##l, edx, eax) \
397 ON64(FOP2E(op##q, rdx, rax)) \
400 #define FOP3E(op, dst, src, src2) \
401 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
403 /* 3-operand, word-only, src2=cl */
404 #define FASTOP3WCL(op) \
407 FOP3E(op##w, ax, dx, cl) \
408 FOP3E(op##l, eax, edx, cl) \
409 ON64(FOP3E(op##q, rax, rdx, cl)) \
412 /* Special case for SETcc - 1 instruction per cc */
413 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
415 asm(".global kvm_fastop_exception \n"
416 "kvm_fastop_exception: xor %esi, %esi; ret");
437 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
441 * XXX: inoutclob user must know where the argument is being expanded.
442 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
444 #define asm_safe(insn, inoutclob...) \
448 asm volatile("1:" insn "\n" \
450 ".pushsection .fixup, \"ax\"\n" \
451 "3: movl $1, %[_fault]\n" \
454 _ASM_EXTABLE(1b, 3b) \
455 : [_fault] "+qm"(_fault) inoutclob ); \
457 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
460 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
461 enum x86_intercept intercept,
462 enum x86_intercept_stage stage)
464 struct x86_instruction_info info = {
465 .intercept = intercept,
466 .rep_prefix = ctxt->rep_prefix,
467 .modrm_mod = ctxt->modrm_mod,
468 .modrm_reg = ctxt->modrm_reg,
469 .modrm_rm = ctxt->modrm_rm,
470 .src_val = ctxt->src.val64,
471 .dst_val = ctxt->dst.val64,
472 .src_bytes = ctxt->src.bytes,
473 .dst_bytes = ctxt->dst.bytes,
474 .ad_bytes = ctxt->ad_bytes,
475 .next_rip = ctxt->eip,
478 return ctxt->ops->intercept(ctxt, &info, stage);
481 static void assign_masked(ulong *dest, ulong src, ulong mask)
483 *dest = (*dest & ~mask) | (src & mask);
486 static void assign_register(unsigned long *reg, u64 val, int bytes)
488 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
491 *(u8 *)reg = (u8)val;
494 *(u16 *)reg = (u16)val;
498 break; /* 64b: zero-extend */
505 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
507 return (1UL << (ctxt->ad_bytes << 3)) - 1;
510 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
513 struct desc_struct ss;
515 if (ctxt->mode == X86EMUL_MODE_PROT64)
517 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
518 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
521 static int stack_size(struct x86_emulate_ctxt *ctxt)
523 return (__fls(stack_mask(ctxt)) + 1) >> 3;
526 /* Access/update address held in a register, based on addressing mode. */
527 static inline unsigned long
528 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
530 if (ctxt->ad_bytes == sizeof(unsigned long))
533 return reg & ad_mask(ctxt);
536 static inline unsigned long
537 register_address(struct x86_emulate_ctxt *ctxt, int reg)
539 return address_mask(ctxt, reg_read(ctxt, reg));
542 static void masked_increment(ulong *reg, ulong mask, int inc)
544 assign_masked(reg, *reg + inc, mask);
548 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
550 ulong *preg = reg_rmw(ctxt, reg);
552 assign_register(preg, *preg + inc, ctxt->ad_bytes);
555 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
557 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
560 static u32 desc_limit_scaled(struct desc_struct *desc)
562 u32 limit = get_desc_limit(desc);
564 return desc->g ? (limit << 12) | 0xfff : limit;
567 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
569 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
572 return ctxt->ops->get_cached_segment_base(ctxt, seg);
575 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
576 u32 error, bool valid)
579 ctxt->exception.vector = vec;
580 ctxt->exception.error_code = error;
581 ctxt->exception.error_code_valid = valid;
582 return X86EMUL_PROPAGATE_FAULT;
585 static int emulate_db(struct x86_emulate_ctxt *ctxt)
587 return emulate_exception(ctxt, DB_VECTOR, 0, false);
590 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
592 return emulate_exception(ctxt, GP_VECTOR, err, true);
595 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
597 return emulate_exception(ctxt, SS_VECTOR, err, true);
600 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
602 return emulate_exception(ctxt, UD_VECTOR, 0, false);
605 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
607 return emulate_exception(ctxt, TS_VECTOR, err, true);
610 static int emulate_de(struct x86_emulate_ctxt *ctxt)
612 return emulate_exception(ctxt, DE_VECTOR, 0, false);
615 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
617 return emulate_exception(ctxt, NM_VECTOR, 0, false);
620 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
623 struct desc_struct desc;
625 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
629 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
634 struct desc_struct desc;
636 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
637 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
641 * x86 defines three classes of vector instructions: explicitly
642 * aligned, explicitly unaligned, and the rest, which change behaviour
643 * depending on whether they're AVX encoded or not.
645 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
646 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
647 * 512 bytes of data must be aligned to a 16 byte boundary.
649 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
651 if (likely(size < 16))
654 if (ctxt->d & Aligned)
656 else if (ctxt->d & Unaligned)
658 else if (ctxt->d & Avx)
660 else if (ctxt->d & Aligned16)
666 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
667 struct segmented_address addr,
668 unsigned *max_size, unsigned size,
669 bool write, bool fetch,
670 enum x86emul_mode mode, ulong *linear)
672 struct desc_struct desc;
678 la = seg_base(ctxt, addr.seg) + addr.ea;
681 case X86EMUL_MODE_PROT64:
683 if (is_noncanonical_address(la))
686 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
687 if (size > *max_size)
691 *linear = la = (u32)la;
692 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
696 /* code segment in protected mode or read-only data segment */
697 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
698 || !(desc.type & 2)) && write)
700 /* unreadable code segment */
701 if (!fetch && (desc.type & 8) && !(desc.type & 2))
703 lim = desc_limit_scaled(&desc);
704 if (!(desc.type & 8) && (desc.type & 4)) {
705 /* expand-down segment */
708 lim = desc.d ? 0xffffffff : 0xffff;
712 if (lim == 0xffffffff)
715 *max_size = (u64)lim + 1 - addr.ea;
716 if (size > *max_size)
721 if (la & (insn_alignment(ctxt, size) - 1))
722 return emulate_gp(ctxt, 0);
723 return X86EMUL_CONTINUE;
725 if (addr.seg == VCPU_SREG_SS)
726 return emulate_ss(ctxt, 0);
728 return emulate_gp(ctxt, 0);
731 static int linearize(struct x86_emulate_ctxt *ctxt,
732 struct segmented_address addr,
733 unsigned size, bool write,
737 return __linearize(ctxt, addr, &max_size, size, write, false,
741 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
742 enum x86emul_mode mode)
747 struct segmented_address addr = { .seg = VCPU_SREG_CS,
750 if (ctxt->op_bytes != sizeof(unsigned long))
751 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
752 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
753 if (rc == X86EMUL_CONTINUE)
754 ctxt->_eip = addr.ea;
758 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
760 return assign_eip(ctxt, dst, ctxt->mode);
763 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
764 const struct desc_struct *cs_desc)
766 enum x86emul_mode mode = ctxt->mode;
770 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
774 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
776 mode = X86EMUL_MODE_PROT64;
778 mode = X86EMUL_MODE_PROT32; /* temporary value */
781 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
782 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
783 rc = assign_eip(ctxt, dst, mode);
784 if (rc == X86EMUL_CONTINUE)
789 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
791 return assign_eip_near(ctxt, ctxt->_eip + rel);
794 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
795 void *data, unsigned size)
797 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
800 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
801 ulong linear, void *data,
804 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
807 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
808 struct segmented_address addr,
815 rc = linearize(ctxt, addr, size, false, &linear);
816 if (rc != X86EMUL_CONTINUE)
818 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
821 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
822 struct segmented_address addr,
829 rc = linearize(ctxt, addr, size, true, &linear);
830 if (rc != X86EMUL_CONTINUE)
832 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
836 * Prefetch the remaining bytes of the instruction without crossing page
837 * boundary if they are not in fetch_cache yet.
839 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
842 unsigned size, max_size;
843 unsigned long linear;
844 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
845 struct segmented_address addr = { .seg = VCPU_SREG_CS,
846 .ea = ctxt->eip + cur_size };
849 * We do not know exactly how many bytes will be needed, and
850 * __linearize is expensive, so fetch as much as possible. We
851 * just have to avoid going beyond the 15 byte limit, the end
852 * of the segment, or the end of the page.
854 * __linearize is called with size 0 so that it does not do any
855 * boundary check itself. Instead, we use max_size to check
858 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
860 if (unlikely(rc != X86EMUL_CONTINUE))
863 size = min_t(unsigned, 15UL ^ cur_size, max_size);
864 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
867 * One instruction can only straddle two pages,
868 * and one has been loaded at the beginning of
869 * x86_decode_insn. So, if not enough bytes
870 * still, we must have hit the 15-byte boundary.
872 if (unlikely(size < op_size))
873 return emulate_gp(ctxt, 0);
875 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
876 size, &ctxt->exception);
877 if (unlikely(rc != X86EMUL_CONTINUE))
879 ctxt->fetch.end += size;
880 return X86EMUL_CONTINUE;
883 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
886 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
888 if (unlikely(done_size < size))
889 return __do_insn_fetch_bytes(ctxt, size - done_size);
891 return X86EMUL_CONTINUE;
894 /* Fetch next part of the instruction being emulated. */
895 #define insn_fetch(_type, _ctxt) \
898 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
899 if (rc != X86EMUL_CONTINUE) \
901 ctxt->_eip += sizeof(_type); \
902 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
903 ctxt->fetch.ptr += sizeof(_type); \
907 #define insn_fetch_arr(_arr, _size, _ctxt) \
909 rc = do_insn_fetch_bytes(_ctxt, _size); \
910 if (rc != X86EMUL_CONTINUE) \
912 ctxt->_eip += (_size); \
913 memcpy(_arr, ctxt->fetch.ptr, _size); \
914 ctxt->fetch.ptr += (_size); \
918 * Given the 'reg' portion of a ModRM byte, and a register block, return a
919 * pointer into the block that addresses the relevant register.
920 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
922 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
926 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
928 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
929 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
931 p = reg_rmw(ctxt, modrm_reg);
935 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
936 struct segmented_address addr,
937 u16 *size, unsigned long *address, int op_bytes)
944 rc = segmented_read_std(ctxt, addr, size, 2);
945 if (rc != X86EMUL_CONTINUE)
948 rc = segmented_read_std(ctxt, addr, address, op_bytes);
962 FASTOP1SRC2(mul, mul_ex);
963 FASTOP1SRC2(imul, imul_ex);
964 FASTOP1SRC2EX(div, div_ex);
965 FASTOP1SRC2EX(idiv, idiv_ex);
994 FASTOP2R(cmp, cmp_r);
996 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
998 /* If src is zero, do not writeback, but update flags */
999 if (ctxt->src.val == 0)
1000 ctxt->dst.type = OP_NONE;
1001 return fastop(ctxt, em_bsf);
1004 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1006 /* If src is zero, do not writeback, but update flags */
1007 if (ctxt->src.val == 0)
1008 ctxt->dst.type = OP_NONE;
1009 return fastop(ctxt, em_bsr);
1012 static u8 test_cc(unsigned int condition, unsigned long flags)
1015 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1017 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1018 asm("push %[flags]; popf; " CALL_NOSPEC
1019 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1023 static void fetch_register_operand(struct operand *op)
1025 switch (op->bytes) {
1027 op->val = *(u8 *)op->addr.reg;
1030 op->val = *(u16 *)op->addr.reg;
1033 op->val = *(u32 *)op->addr.reg;
1036 op->val = *(u64 *)op->addr.reg;
1041 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1043 ctxt->ops->get_fpu(ctxt);
1045 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1046 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1047 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1048 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1049 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1050 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1051 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1052 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1053 #ifdef CONFIG_X86_64
1054 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1055 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1056 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1057 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1058 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1059 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1060 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1061 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1065 ctxt->ops->put_fpu(ctxt);
1068 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1071 ctxt->ops->get_fpu(ctxt);
1073 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1074 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1075 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1076 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1077 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1078 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1079 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1080 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1081 #ifdef CONFIG_X86_64
1082 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1083 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1084 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1085 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1086 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1087 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1088 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1089 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1093 ctxt->ops->put_fpu(ctxt);
1096 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1098 ctxt->ops->get_fpu(ctxt);
1100 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1101 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1102 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1103 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1104 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1105 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1106 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1107 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1110 ctxt->ops->put_fpu(ctxt);
1113 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1115 ctxt->ops->get_fpu(ctxt);
1117 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1118 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1119 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1120 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1121 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1122 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1123 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1124 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1127 ctxt->ops->put_fpu(ctxt);
1130 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1132 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1133 return emulate_nm(ctxt);
1135 ctxt->ops->get_fpu(ctxt);
1136 asm volatile("fninit");
1137 ctxt->ops->put_fpu(ctxt);
1138 return X86EMUL_CONTINUE;
1141 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1145 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1146 return emulate_nm(ctxt);
1148 ctxt->ops->get_fpu(ctxt);
1149 asm volatile("fnstcw %0": "+m"(fcw));
1150 ctxt->ops->put_fpu(ctxt);
1152 ctxt->dst.val = fcw;
1154 return X86EMUL_CONTINUE;
1157 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1161 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1162 return emulate_nm(ctxt);
1164 ctxt->ops->get_fpu(ctxt);
1165 asm volatile("fnstsw %0": "+m"(fsw));
1166 ctxt->ops->put_fpu(ctxt);
1168 ctxt->dst.val = fsw;
1170 return X86EMUL_CONTINUE;
1173 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1176 unsigned reg = ctxt->modrm_reg;
1178 if (!(ctxt->d & ModRM))
1179 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1181 if (ctxt->d & Sse) {
1185 read_sse_reg(ctxt, &op->vec_val, reg);
1188 if (ctxt->d & Mmx) {
1197 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1198 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1200 fetch_register_operand(op);
1201 op->orig_val = op->val;
1204 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1206 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1207 ctxt->modrm_seg = VCPU_SREG_SS;
1210 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1214 int index_reg, base_reg, scale;
1215 int rc = X86EMUL_CONTINUE;
1218 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1219 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1220 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1222 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1223 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1224 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1225 ctxt->modrm_seg = VCPU_SREG_DS;
1227 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1229 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1230 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1232 if (ctxt->d & Sse) {
1235 op->addr.xmm = ctxt->modrm_rm;
1236 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1239 if (ctxt->d & Mmx) {
1242 op->addr.mm = ctxt->modrm_rm & 7;
1245 fetch_register_operand(op);
1251 if (ctxt->ad_bytes == 2) {
1252 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1253 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1254 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1255 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1257 /* 16-bit ModR/M decode. */
1258 switch (ctxt->modrm_mod) {
1260 if (ctxt->modrm_rm == 6)
1261 modrm_ea += insn_fetch(u16, ctxt);
1264 modrm_ea += insn_fetch(s8, ctxt);
1267 modrm_ea += insn_fetch(u16, ctxt);
1270 switch (ctxt->modrm_rm) {
1272 modrm_ea += bx + si;
1275 modrm_ea += bx + di;
1278 modrm_ea += bp + si;
1281 modrm_ea += bp + di;
1290 if (ctxt->modrm_mod != 0)
1297 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1298 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1299 ctxt->modrm_seg = VCPU_SREG_SS;
1300 modrm_ea = (u16)modrm_ea;
1302 /* 32/64-bit ModR/M decode. */
1303 if ((ctxt->modrm_rm & 7) == 4) {
1304 sib = insn_fetch(u8, ctxt);
1305 index_reg |= (sib >> 3) & 7;
1306 base_reg |= sib & 7;
1309 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1310 modrm_ea += insn_fetch(s32, ctxt);
1312 modrm_ea += reg_read(ctxt, base_reg);
1313 adjust_modrm_seg(ctxt, base_reg);
1314 /* Increment ESP on POP [ESP] */
1315 if ((ctxt->d & IncSP) &&
1316 base_reg == VCPU_REGS_RSP)
1317 modrm_ea += ctxt->op_bytes;
1320 modrm_ea += reg_read(ctxt, index_reg) << scale;
1321 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1322 modrm_ea += insn_fetch(s32, ctxt);
1323 if (ctxt->mode == X86EMUL_MODE_PROT64)
1324 ctxt->rip_relative = 1;
1326 base_reg = ctxt->modrm_rm;
1327 modrm_ea += reg_read(ctxt, base_reg);
1328 adjust_modrm_seg(ctxt, base_reg);
1330 switch (ctxt->modrm_mod) {
1332 modrm_ea += insn_fetch(s8, ctxt);
1335 modrm_ea += insn_fetch(s32, ctxt);
1339 op->addr.mem.ea = modrm_ea;
1340 if (ctxt->ad_bytes != 8)
1341 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1347 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1350 int rc = X86EMUL_CONTINUE;
1353 switch (ctxt->ad_bytes) {
1355 op->addr.mem.ea = insn_fetch(u16, ctxt);
1358 op->addr.mem.ea = insn_fetch(u32, ctxt);
1361 op->addr.mem.ea = insn_fetch(u64, ctxt);
1368 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1372 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1373 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1375 if (ctxt->src.bytes == 2)
1376 sv = (s16)ctxt->src.val & (s16)mask;
1377 else if (ctxt->src.bytes == 4)
1378 sv = (s32)ctxt->src.val & (s32)mask;
1380 sv = (s64)ctxt->src.val & (s64)mask;
1382 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1383 ctxt->dst.addr.mem.ea + (sv >> 3));
1386 /* only subword offset */
1387 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1390 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1391 unsigned long addr, void *dest, unsigned size)
1394 struct read_cache *mc = &ctxt->mem_read;
1396 if (mc->pos < mc->end)
1399 WARN_ON((mc->end + size) >= sizeof(mc->data));
1401 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1403 if (rc != X86EMUL_CONTINUE)
1409 memcpy(dest, mc->data + mc->pos, size);
1411 return X86EMUL_CONTINUE;
1414 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1415 struct segmented_address addr,
1422 rc = linearize(ctxt, addr, size, false, &linear);
1423 if (rc != X86EMUL_CONTINUE)
1425 return read_emulated(ctxt, linear, data, size);
1428 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1429 struct segmented_address addr,
1436 rc = linearize(ctxt, addr, size, true, &linear);
1437 if (rc != X86EMUL_CONTINUE)
1439 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1443 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1444 struct segmented_address addr,
1445 const void *orig_data, const void *data,
1451 rc = linearize(ctxt, addr, size, true, &linear);
1452 if (rc != X86EMUL_CONTINUE)
1454 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1455 size, &ctxt->exception);
1458 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1459 unsigned int size, unsigned short port,
1462 struct read_cache *rc = &ctxt->io_read;
1464 if (rc->pos == rc->end) { /* refill pio read ahead */
1465 unsigned int in_page, n;
1466 unsigned int count = ctxt->rep_prefix ?
1467 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1468 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1469 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1470 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1471 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1474 rc->pos = rc->end = 0;
1475 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1480 if (ctxt->rep_prefix && (ctxt->d & String) &&
1481 !(ctxt->eflags & X86_EFLAGS_DF)) {
1482 ctxt->dst.data = rc->data + rc->pos;
1483 ctxt->dst.type = OP_MEM_STR;
1484 ctxt->dst.count = (rc->end - rc->pos) / size;
1487 memcpy(dest, rc->data + rc->pos, size);
1493 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1494 u16 index, struct desc_struct *desc)
1499 ctxt->ops->get_idt(ctxt, &dt);
1501 if (dt.size < index * 8 + 7)
1502 return emulate_gp(ctxt, index << 3 | 0x2);
1504 addr = dt.address + index * 8;
1505 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1508 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1509 u16 selector, struct desc_ptr *dt)
1511 const struct x86_emulate_ops *ops = ctxt->ops;
1514 if (selector & 1 << 2) {
1515 struct desc_struct desc;
1518 memset (dt, 0, sizeof *dt);
1519 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1523 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1524 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1526 ops->get_gdt(ctxt, dt);
1529 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1530 u16 selector, ulong *desc_addr_p)
1533 u16 index = selector >> 3;
1536 get_descriptor_table_ptr(ctxt, selector, &dt);
1538 if (dt.size < index * 8 + 7)
1539 return emulate_gp(ctxt, selector & 0xfffc);
1541 addr = dt.address + index * 8;
1543 #ifdef CONFIG_X86_64
1544 if (addr >> 32 != 0) {
1547 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1548 if (!(efer & EFER_LMA))
1553 *desc_addr_p = addr;
1554 return X86EMUL_CONTINUE;
1557 /* allowed just for 8 bytes segments */
1558 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1559 u16 selector, struct desc_struct *desc,
1564 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1565 if (rc != X86EMUL_CONTINUE)
1568 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1571 /* allowed just for 8 bytes segments */
1572 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1573 u16 selector, struct desc_struct *desc)
1578 rc = get_descriptor_ptr(ctxt, selector, &addr);
1579 if (rc != X86EMUL_CONTINUE)
1582 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1585 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1586 u16 selector, int seg, u8 cpl,
1587 enum x86_transfer_type transfer,
1588 struct desc_struct *desc)
1590 struct desc_struct seg_desc, old_desc;
1592 unsigned err_vec = GP_VECTOR;
1594 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1600 memset(&seg_desc, 0, sizeof seg_desc);
1602 if (ctxt->mode == X86EMUL_MODE_REAL) {
1603 /* set real mode segment descriptor (keep limit etc. for
1605 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1606 set_desc_base(&seg_desc, selector << 4);
1608 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1609 /* VM86 needs a clean new segment descriptor */
1610 set_desc_base(&seg_desc, selector << 4);
1611 set_desc_limit(&seg_desc, 0xffff);
1621 /* TR should be in GDT only */
1622 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1625 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1626 if (null_selector) {
1627 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1630 if (seg == VCPU_SREG_SS) {
1631 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1635 * ctxt->ops->set_segment expects the CPL to be in
1636 * SS.DPL, so fake an expand-up 32-bit data segment.
1646 /* Skip all following checks */
1650 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1651 if (ret != X86EMUL_CONTINUE)
1654 err_code = selector & 0xfffc;
1655 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1658 /* can't load system descriptor into segment selector */
1659 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1660 if (transfer == X86_TRANSFER_CALL_JMP)
1661 return X86EMUL_UNHANDLEABLE;
1666 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1675 * segment is not a writable data segment or segment
1676 * selector's RPL != CPL or segment selector's RPL != CPL
1678 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1682 if (!(seg_desc.type & 8))
1685 if (seg_desc.type & 4) {
1691 if (rpl > cpl || dpl != cpl)
1694 /* in long-mode d/b must be clear if l is set */
1695 if (seg_desc.d && seg_desc.l) {
1698 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1699 if (efer & EFER_LMA)
1703 /* CS(RPL) <- CPL */
1704 selector = (selector & 0xfffc) | cpl;
1707 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1709 old_desc = seg_desc;
1710 seg_desc.type |= 2; /* busy */
1711 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1712 sizeof(seg_desc), &ctxt->exception);
1713 if (ret != X86EMUL_CONTINUE)
1716 case VCPU_SREG_LDTR:
1717 if (seg_desc.s || seg_desc.type != 2)
1720 default: /* DS, ES, FS, or GS */
1722 * segment is not a data or readable code segment or
1723 * ((segment is a data or nonconforming code segment)
1724 * and (both RPL and CPL > DPL))
1726 if ((seg_desc.type & 0xa) == 0x8 ||
1727 (((seg_desc.type & 0xc) != 0xc) &&
1728 (rpl > dpl && cpl > dpl)))
1734 /* mark segment as accessed */
1735 if (!(seg_desc.type & 1)) {
1737 ret = write_segment_descriptor(ctxt, selector,
1739 if (ret != X86EMUL_CONTINUE)
1742 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1743 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1744 if (ret != X86EMUL_CONTINUE)
1746 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1747 ((u64)base3 << 32)))
1748 return emulate_gp(ctxt, 0);
1751 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1754 return X86EMUL_CONTINUE;
1756 return emulate_exception(ctxt, err_vec, err_code, true);
1759 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1760 u16 selector, int seg)
1762 u8 cpl = ctxt->ops->cpl(ctxt);
1765 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1766 * they can load it at CPL<3 (Intel's manual says only LSS can,
1769 * However, the Intel manual says that putting IST=1/DPL=3 in
1770 * an interrupt gate will result in SS=3 (the AMD manual instead
1771 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1772 * and only forbid it here.
1774 if (seg == VCPU_SREG_SS && selector == 3 &&
1775 ctxt->mode == X86EMUL_MODE_PROT64)
1776 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1778 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1779 X86_TRANSFER_NONE, NULL);
1782 static void write_register_operand(struct operand *op)
1784 return assign_register(op->addr.reg, op->val, op->bytes);
1787 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1791 write_register_operand(op);
1794 if (ctxt->lock_prefix)
1795 return segmented_cmpxchg(ctxt,
1801 return segmented_write(ctxt,
1807 return segmented_write(ctxt,
1810 op->bytes * op->count);
1813 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1816 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1824 return X86EMUL_CONTINUE;
1827 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1829 struct segmented_address addr;
1831 rsp_increment(ctxt, -bytes);
1832 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1833 addr.seg = VCPU_SREG_SS;
1835 return segmented_write(ctxt, addr, data, bytes);
1838 static int em_push(struct x86_emulate_ctxt *ctxt)
1840 /* Disable writeback. */
1841 ctxt->dst.type = OP_NONE;
1842 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1845 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1846 void *dest, int len)
1849 struct segmented_address addr;
1851 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1852 addr.seg = VCPU_SREG_SS;
1853 rc = segmented_read(ctxt, addr, dest, len);
1854 if (rc != X86EMUL_CONTINUE)
1857 rsp_increment(ctxt, len);
1861 static int em_pop(struct x86_emulate_ctxt *ctxt)
1863 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1866 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1867 void *dest, int len)
1870 unsigned long val, change_mask;
1871 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1872 int cpl = ctxt->ops->cpl(ctxt);
1874 rc = emulate_pop(ctxt, &val, len);
1875 if (rc != X86EMUL_CONTINUE)
1878 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1879 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1880 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1881 X86_EFLAGS_AC | X86_EFLAGS_ID;
1883 switch(ctxt->mode) {
1884 case X86EMUL_MODE_PROT64:
1885 case X86EMUL_MODE_PROT32:
1886 case X86EMUL_MODE_PROT16:
1888 change_mask |= X86_EFLAGS_IOPL;
1890 change_mask |= X86_EFLAGS_IF;
1892 case X86EMUL_MODE_VM86:
1894 return emulate_gp(ctxt, 0);
1895 change_mask |= X86_EFLAGS_IF;
1897 default: /* real mode */
1898 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1902 *(unsigned long *)dest =
1903 (ctxt->eflags & ~change_mask) | (val & change_mask);
1908 static int em_popf(struct x86_emulate_ctxt *ctxt)
1910 ctxt->dst.type = OP_REG;
1911 ctxt->dst.addr.reg = &ctxt->eflags;
1912 ctxt->dst.bytes = ctxt->op_bytes;
1913 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1916 static int em_enter(struct x86_emulate_ctxt *ctxt)
1919 unsigned frame_size = ctxt->src.val;
1920 unsigned nesting_level = ctxt->src2.val & 31;
1924 return X86EMUL_UNHANDLEABLE;
1926 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1927 rc = push(ctxt, &rbp, stack_size(ctxt));
1928 if (rc != X86EMUL_CONTINUE)
1930 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1932 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1933 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1935 return X86EMUL_CONTINUE;
1938 static int em_leave(struct x86_emulate_ctxt *ctxt)
1940 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1942 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1945 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1947 int seg = ctxt->src2.val;
1949 ctxt->src.val = get_segment_selector(ctxt, seg);
1950 if (ctxt->op_bytes == 4) {
1951 rsp_increment(ctxt, -2);
1955 return em_push(ctxt);
1958 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1960 int seg = ctxt->src2.val;
1961 unsigned long selector;
1964 rc = emulate_pop(ctxt, &selector, 2);
1965 if (rc != X86EMUL_CONTINUE)
1968 if (ctxt->modrm_reg == VCPU_SREG_SS)
1969 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1970 if (ctxt->op_bytes > 2)
1971 rsp_increment(ctxt, ctxt->op_bytes - 2);
1973 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1977 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1979 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1980 int rc = X86EMUL_CONTINUE;
1981 int reg = VCPU_REGS_RAX;
1983 while (reg <= VCPU_REGS_RDI) {
1984 (reg == VCPU_REGS_RSP) ?
1985 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1988 if (rc != X86EMUL_CONTINUE)
1997 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1999 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2000 return em_push(ctxt);
2003 static int em_popa(struct x86_emulate_ctxt *ctxt)
2005 int rc = X86EMUL_CONTINUE;
2006 int reg = VCPU_REGS_RDI;
2009 while (reg >= VCPU_REGS_RAX) {
2010 if (reg == VCPU_REGS_RSP) {
2011 rsp_increment(ctxt, ctxt->op_bytes);
2015 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2016 if (rc != X86EMUL_CONTINUE)
2018 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2024 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2026 const struct x86_emulate_ops *ops = ctxt->ops;
2033 /* TODO: Add limit checks */
2034 ctxt->src.val = ctxt->eflags;
2036 if (rc != X86EMUL_CONTINUE)
2039 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2041 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2043 if (rc != X86EMUL_CONTINUE)
2046 ctxt->src.val = ctxt->_eip;
2048 if (rc != X86EMUL_CONTINUE)
2051 ops->get_idt(ctxt, &dt);
2053 eip_addr = dt.address + (irq << 2);
2054 cs_addr = dt.address + (irq << 2) + 2;
2056 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2057 if (rc != X86EMUL_CONTINUE)
2060 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2061 if (rc != X86EMUL_CONTINUE)
2064 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2065 if (rc != X86EMUL_CONTINUE)
2073 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2077 invalidate_registers(ctxt);
2078 rc = __emulate_int_real(ctxt, irq);
2079 if (rc == X86EMUL_CONTINUE)
2080 writeback_registers(ctxt);
2084 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2086 switch(ctxt->mode) {
2087 case X86EMUL_MODE_REAL:
2088 return __emulate_int_real(ctxt, irq);
2089 case X86EMUL_MODE_VM86:
2090 case X86EMUL_MODE_PROT16:
2091 case X86EMUL_MODE_PROT32:
2092 case X86EMUL_MODE_PROT64:
2094 /* Protected mode interrupts unimplemented yet */
2095 return X86EMUL_UNHANDLEABLE;
2099 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2101 int rc = X86EMUL_CONTINUE;
2102 unsigned long temp_eip = 0;
2103 unsigned long temp_eflags = 0;
2104 unsigned long cs = 0;
2105 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2106 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2107 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2108 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2109 X86_EFLAGS_AC | X86_EFLAGS_ID |
2111 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2114 /* TODO: Add stack limit check */
2116 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2118 if (rc != X86EMUL_CONTINUE)
2121 if (temp_eip & ~0xffff)
2122 return emulate_gp(ctxt, 0);
2124 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2126 if (rc != X86EMUL_CONTINUE)
2129 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2131 if (rc != X86EMUL_CONTINUE)
2134 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2136 if (rc != X86EMUL_CONTINUE)
2139 ctxt->_eip = temp_eip;
2141 if (ctxt->op_bytes == 4)
2142 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2143 else if (ctxt->op_bytes == 2) {
2144 ctxt->eflags &= ~0xffff;
2145 ctxt->eflags |= temp_eflags;
2148 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2149 ctxt->eflags |= X86_EFLAGS_FIXED;
2150 ctxt->ops->set_nmi_mask(ctxt, false);
2155 static int em_iret(struct x86_emulate_ctxt *ctxt)
2157 switch(ctxt->mode) {
2158 case X86EMUL_MODE_REAL:
2159 return emulate_iret_real(ctxt);
2160 case X86EMUL_MODE_VM86:
2161 case X86EMUL_MODE_PROT16:
2162 case X86EMUL_MODE_PROT32:
2163 case X86EMUL_MODE_PROT64:
2165 /* iret from protected mode unimplemented yet */
2166 return X86EMUL_UNHANDLEABLE;
2170 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2174 struct desc_struct new_desc;
2175 u8 cpl = ctxt->ops->cpl(ctxt);
2177 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2179 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2180 X86_TRANSFER_CALL_JMP,
2182 if (rc != X86EMUL_CONTINUE)
2185 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2186 /* Error handling is not implemented. */
2187 if (rc != X86EMUL_CONTINUE)
2188 return X86EMUL_UNHANDLEABLE;
2193 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2195 return assign_eip_near(ctxt, ctxt->src.val);
2198 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2203 old_eip = ctxt->_eip;
2204 rc = assign_eip_near(ctxt, ctxt->src.val);
2205 if (rc != X86EMUL_CONTINUE)
2207 ctxt->src.val = old_eip;
2212 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2214 u64 old = ctxt->dst.orig_val64;
2216 if (ctxt->dst.bytes == 16)
2217 return X86EMUL_UNHANDLEABLE;
2219 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2220 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2221 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2222 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2223 ctxt->eflags &= ~X86_EFLAGS_ZF;
2225 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2226 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2228 ctxt->eflags |= X86_EFLAGS_ZF;
2230 return X86EMUL_CONTINUE;
2233 static int em_ret(struct x86_emulate_ctxt *ctxt)
2238 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2239 if (rc != X86EMUL_CONTINUE)
2242 return assign_eip_near(ctxt, eip);
2245 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2248 unsigned long eip, cs;
2249 int cpl = ctxt->ops->cpl(ctxt);
2250 struct desc_struct new_desc;
2252 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2253 if (rc != X86EMUL_CONTINUE)
2255 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2256 if (rc != X86EMUL_CONTINUE)
2258 /* Outer-privilege level return is not implemented */
2259 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2260 return X86EMUL_UNHANDLEABLE;
2261 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2264 if (rc != X86EMUL_CONTINUE)
2266 rc = assign_eip_far(ctxt, eip, &new_desc);
2267 /* Error handling is not implemented. */
2268 if (rc != X86EMUL_CONTINUE)
2269 return X86EMUL_UNHANDLEABLE;
2274 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2278 rc = em_ret_far(ctxt);
2279 if (rc != X86EMUL_CONTINUE)
2281 rsp_increment(ctxt, ctxt->src.val);
2282 return X86EMUL_CONTINUE;
2285 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2287 /* Save real source value, then compare EAX against destination. */
2288 ctxt->dst.orig_val = ctxt->dst.val;
2289 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2290 ctxt->src.orig_val = ctxt->src.val;
2291 ctxt->src.val = ctxt->dst.orig_val;
2292 fastop(ctxt, em_cmp);
2294 if (ctxt->eflags & X86_EFLAGS_ZF) {
2295 /* Success: write back to memory; no update of EAX */
2296 ctxt->src.type = OP_NONE;
2297 ctxt->dst.val = ctxt->src.orig_val;
2299 /* Failure: write the value we saw to EAX. */
2300 ctxt->src.type = OP_REG;
2301 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2302 ctxt->src.val = ctxt->dst.orig_val;
2303 /* Create write-cycle to dest by writing the same value */
2304 ctxt->dst.val = ctxt->dst.orig_val;
2306 return X86EMUL_CONTINUE;
2309 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2311 int seg = ctxt->src2.val;
2315 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2317 rc = load_segment_descriptor(ctxt, sel, seg);
2318 if (rc != X86EMUL_CONTINUE)
2321 ctxt->dst.val = ctxt->src.val;
2325 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2327 u32 eax, ebx, ecx, edx;
2331 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2332 return edx & bit(X86_FEATURE_LM);
2335 #define GET_SMSTATE(type, smbase, offset) \
2338 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2340 if (r != X86EMUL_CONTINUE) \
2341 return X86EMUL_UNHANDLEABLE; \
2345 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2347 desc->g = (flags >> 23) & 1;
2348 desc->d = (flags >> 22) & 1;
2349 desc->l = (flags >> 21) & 1;
2350 desc->avl = (flags >> 20) & 1;
2351 desc->p = (flags >> 15) & 1;
2352 desc->dpl = (flags >> 13) & 3;
2353 desc->s = (flags >> 12) & 1;
2354 desc->type = (flags >> 8) & 15;
2357 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2359 struct desc_struct desc;
2363 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2366 offset = 0x7f84 + n * 12;
2368 offset = 0x7f2c + (n - 3) * 12;
2370 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2371 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2372 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2373 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2374 return X86EMUL_CONTINUE;
2377 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2379 struct desc_struct desc;
2384 offset = 0x7e00 + n * 16;
2386 selector = GET_SMSTATE(u16, smbase, offset);
2387 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2388 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2389 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2390 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2392 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2393 return X86EMUL_CONTINUE;
2396 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2397 u64 cr0, u64 cr3, u64 cr4)
2402 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2404 if (cr4 & X86_CR4_PCIDE) {
2409 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2411 return X86EMUL_UNHANDLEABLE;
2414 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2415 * Then enable protected mode. However, PCID cannot be enabled
2416 * if EFER.LMA=0, so set it separately.
2418 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2420 return X86EMUL_UNHANDLEABLE;
2422 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2424 return X86EMUL_UNHANDLEABLE;
2426 if (cr4 & X86_CR4_PCIDE) {
2427 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2429 return X86EMUL_UNHANDLEABLE;
2431 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2433 return X86EMUL_UNHANDLEABLE;
2438 return X86EMUL_CONTINUE;
2441 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2443 struct desc_struct desc;
2446 u32 val, cr0, cr3, cr4;
2449 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2450 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2451 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2452 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2454 for (i = 0; i < 8; i++)
2455 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2457 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2458 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2459 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2460 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2462 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2463 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2464 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2465 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2466 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2468 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2469 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2470 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2471 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2472 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2474 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2475 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2476 ctxt->ops->set_gdt(ctxt, &dt);
2478 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2479 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2480 ctxt->ops->set_idt(ctxt, &dt);
2482 for (i = 0; i < 6; i++) {
2483 int r = rsm_load_seg_32(ctxt, smbase, i);
2484 if (r != X86EMUL_CONTINUE)
2488 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2490 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2492 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2495 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2497 struct desc_struct desc;
2499 u64 val, cr0, cr3, cr4;
2504 for (i = 0; i < 16; i++)
2505 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2507 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2508 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2510 val = GET_SMSTATE(u32, smbase, 0x7f68);
2511 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2512 val = GET_SMSTATE(u32, smbase, 0x7f60);
2513 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2515 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2516 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2517 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2518 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2519 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2520 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2522 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2523 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2524 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2525 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2526 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2527 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2529 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2530 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2531 ctxt->ops->set_idt(ctxt, &dt);
2533 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2534 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2535 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2536 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2537 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2538 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2540 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2541 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2542 ctxt->ops->set_gdt(ctxt, &dt);
2544 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2545 if (r != X86EMUL_CONTINUE)
2548 for (i = 0; i < 6; i++) {
2549 r = rsm_load_seg_64(ctxt, smbase, i);
2550 if (r != X86EMUL_CONTINUE)
2554 return X86EMUL_CONTINUE;
2557 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2559 unsigned long cr0, cr4, efer;
2563 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2564 return emulate_ud(ctxt);
2567 * Get back to real mode, to prepare a safe state in which to load
2568 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2569 * supports long mode.
2571 if (emulator_has_longmode(ctxt)) {
2572 struct desc_struct cs_desc;
2574 /* Zero CR4.PCIDE before CR0.PG. */
2575 cr4 = ctxt->ops->get_cr(ctxt, 4);
2576 if (cr4 & X86_CR4_PCIDE)
2577 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2579 /* A 32-bit code segment is required to clear EFER.LMA. */
2580 memset(&cs_desc, 0, sizeof(cs_desc));
2582 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2583 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2586 /* For the 64-bit case, this will clear EFER.LMA. */
2587 cr0 = ctxt->ops->get_cr(ctxt, 0);
2588 if (cr0 & X86_CR0_PE)
2589 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2591 if (emulator_has_longmode(ctxt)) {
2592 /* Clear CR4.PAE before clearing EFER.LME. */
2593 cr4 = ctxt->ops->get_cr(ctxt, 4);
2594 if (cr4 & X86_CR4_PAE)
2595 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2597 /* And finally go back to 32-bit mode. */
2599 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2602 smbase = ctxt->ops->get_smbase(ctxt);
2603 if (emulator_has_longmode(ctxt))
2604 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2606 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2608 if (ret != X86EMUL_CONTINUE) {
2609 /* FIXME: should triple fault */
2610 return X86EMUL_UNHANDLEABLE;
2613 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2614 ctxt->ops->set_nmi_mask(ctxt, false);
2616 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2617 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2618 return X86EMUL_CONTINUE;
2622 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2623 struct desc_struct *cs, struct desc_struct *ss)
2625 cs->l = 0; /* will be adjusted later */
2626 set_desc_base(cs, 0); /* flat segment */
2627 cs->g = 1; /* 4kb granularity */
2628 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2629 cs->type = 0x0b; /* Read, Execute, Accessed */
2631 cs->dpl = 0; /* will be adjusted later */
2636 set_desc_base(ss, 0); /* flat segment */
2637 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2638 ss->g = 1; /* 4kb granularity */
2640 ss->type = 0x03; /* Read/Write, Accessed */
2641 ss->d = 1; /* 32bit stack segment */
2648 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2650 u32 eax, ebx, ecx, edx;
2653 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2654 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2655 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2656 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2659 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2661 const struct x86_emulate_ops *ops = ctxt->ops;
2662 u32 eax, ebx, ecx, edx;
2665 * syscall should always be enabled in longmode - so only become
2666 * vendor specific (cpuid) if other modes are active...
2668 if (ctxt->mode == X86EMUL_MODE_PROT64)
2673 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2675 * Intel ("GenuineIntel")
2676 * remark: Intel CPUs only support "syscall" in 64bit
2677 * longmode. Also an 64bit guest with a
2678 * 32bit compat-app running will #UD !! While this
2679 * behaviour can be fixed (by emulating) into AMD
2680 * response - CPUs of AMD can't behave like Intel.
2682 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2683 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2684 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2687 /* AMD ("AuthenticAMD") */
2688 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2689 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2690 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2693 /* AMD ("AMDisbetter!") */
2694 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2695 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2696 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2699 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2703 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2705 const struct x86_emulate_ops *ops = ctxt->ops;
2706 struct desc_struct cs, ss;
2711 /* syscall is not available in real mode */
2712 if (ctxt->mode == X86EMUL_MODE_REAL ||
2713 ctxt->mode == X86EMUL_MODE_VM86)
2714 return emulate_ud(ctxt);
2716 if (!(em_syscall_is_enabled(ctxt)))
2717 return emulate_ud(ctxt);
2719 ops->get_msr(ctxt, MSR_EFER, &efer);
2720 setup_syscalls_segments(ctxt, &cs, &ss);
2722 if (!(efer & EFER_SCE))
2723 return emulate_ud(ctxt);
2725 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2727 cs_sel = (u16)(msr_data & 0xfffc);
2728 ss_sel = (u16)(msr_data + 8);
2730 if (efer & EFER_LMA) {
2734 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2735 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2737 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2738 if (efer & EFER_LMA) {
2739 #ifdef CONFIG_X86_64
2740 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2743 ctxt->mode == X86EMUL_MODE_PROT64 ?
2744 MSR_LSTAR : MSR_CSTAR, &msr_data);
2745 ctxt->_eip = msr_data;
2747 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2748 ctxt->eflags &= ~msr_data;
2749 ctxt->eflags |= X86_EFLAGS_FIXED;
2753 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2754 ctxt->_eip = (u32)msr_data;
2756 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2759 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2760 return X86EMUL_CONTINUE;
2763 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2765 const struct x86_emulate_ops *ops = ctxt->ops;
2766 struct desc_struct cs, ss;
2771 ops->get_msr(ctxt, MSR_EFER, &efer);
2772 /* inject #GP if in real mode */
2773 if (ctxt->mode == X86EMUL_MODE_REAL)
2774 return emulate_gp(ctxt, 0);
2777 * Not recognized on AMD in compat mode (but is recognized in legacy
2780 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2781 && !vendor_intel(ctxt))
2782 return emulate_ud(ctxt);
2784 /* sysenter/sysexit have not been tested in 64bit mode. */
2785 if (ctxt->mode == X86EMUL_MODE_PROT64)
2786 return X86EMUL_UNHANDLEABLE;
2788 setup_syscalls_segments(ctxt, &cs, &ss);
2790 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2791 if ((msr_data & 0xfffc) == 0x0)
2792 return emulate_gp(ctxt, 0);
2794 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2795 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2796 ss_sel = cs_sel + 8;
2797 if (efer & EFER_LMA) {
2802 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2803 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2805 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2806 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2808 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2809 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2812 return X86EMUL_CONTINUE;
2815 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2817 const struct x86_emulate_ops *ops = ctxt->ops;
2818 struct desc_struct cs, ss;
2819 u64 msr_data, rcx, rdx;
2821 u16 cs_sel = 0, ss_sel = 0;
2823 /* inject #GP if in real mode or Virtual 8086 mode */
2824 if (ctxt->mode == X86EMUL_MODE_REAL ||
2825 ctxt->mode == X86EMUL_MODE_VM86)
2826 return emulate_gp(ctxt, 0);
2828 setup_syscalls_segments(ctxt, &cs, &ss);
2830 if ((ctxt->rex_prefix & 0x8) != 0x0)
2831 usermode = X86EMUL_MODE_PROT64;
2833 usermode = X86EMUL_MODE_PROT32;
2835 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2836 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2840 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2842 case X86EMUL_MODE_PROT32:
2843 cs_sel = (u16)(msr_data + 16);
2844 if ((msr_data & 0xfffc) == 0x0)
2845 return emulate_gp(ctxt, 0);
2846 ss_sel = (u16)(msr_data + 24);
2850 case X86EMUL_MODE_PROT64:
2851 cs_sel = (u16)(msr_data + 32);
2852 if (msr_data == 0x0)
2853 return emulate_gp(ctxt, 0);
2854 ss_sel = cs_sel + 8;
2857 if (is_noncanonical_address(rcx) ||
2858 is_noncanonical_address(rdx))
2859 return emulate_gp(ctxt, 0);
2862 cs_sel |= SEGMENT_RPL_MASK;
2863 ss_sel |= SEGMENT_RPL_MASK;
2865 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2866 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2869 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2871 return X86EMUL_CONTINUE;
2874 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2877 if (ctxt->mode == X86EMUL_MODE_REAL)
2879 if (ctxt->mode == X86EMUL_MODE_VM86)
2881 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2882 return ctxt->ops->cpl(ctxt) > iopl;
2885 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2888 const struct x86_emulate_ops *ops = ctxt->ops;
2889 struct desc_struct tr_seg;
2892 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2893 unsigned mask = (1 << len) - 1;
2896 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2899 if (desc_limit_scaled(&tr_seg) < 103)
2901 base = get_desc_base(&tr_seg);
2902 #ifdef CONFIG_X86_64
2903 base |= ((u64)base3) << 32;
2905 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2906 if (r != X86EMUL_CONTINUE)
2908 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2910 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2911 if (r != X86EMUL_CONTINUE)
2913 if ((perm >> bit_idx) & mask)
2918 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2924 if (emulator_bad_iopl(ctxt))
2925 if (!emulator_io_port_access_allowed(ctxt, port, len))
2928 ctxt->perm_ok = true;
2933 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2936 * Intel CPUs mask the counter and pointers in quite strange
2937 * manner when ECX is zero due to REP-string optimizations.
2939 #ifdef CONFIG_X86_64
2940 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2943 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2946 case 0xa4: /* movsb */
2947 case 0xa5: /* movsd/w */
2948 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2950 case 0xaa: /* stosb */
2951 case 0xab: /* stosd/w */
2952 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2957 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2958 struct tss_segment_16 *tss)
2960 tss->ip = ctxt->_eip;
2961 tss->flag = ctxt->eflags;
2962 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2963 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2964 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2965 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2966 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2967 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2968 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2969 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2971 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2972 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2973 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2974 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2975 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2978 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2979 struct tss_segment_16 *tss)
2984 ctxt->_eip = tss->ip;
2985 ctxt->eflags = tss->flag | 2;
2986 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2987 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2988 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2989 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2990 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2991 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2992 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2993 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2996 * SDM says that segment selectors are loaded before segment
2999 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3000 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3001 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3002 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3003 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3008 * Now load segment descriptors. If fault happens at this stage
3009 * it is handled in a context of new task
3011 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3012 X86_TRANSFER_TASK_SWITCH, NULL);
3013 if (ret != X86EMUL_CONTINUE)
3015 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3016 X86_TRANSFER_TASK_SWITCH, NULL);
3017 if (ret != X86EMUL_CONTINUE)
3019 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3020 X86_TRANSFER_TASK_SWITCH, NULL);
3021 if (ret != X86EMUL_CONTINUE)
3023 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3024 X86_TRANSFER_TASK_SWITCH, NULL);
3025 if (ret != X86EMUL_CONTINUE)
3027 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3028 X86_TRANSFER_TASK_SWITCH, NULL);
3029 if (ret != X86EMUL_CONTINUE)
3032 return X86EMUL_CONTINUE;
3035 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3036 u16 tss_selector, u16 old_tss_sel,
3037 ulong old_tss_base, struct desc_struct *new_desc)
3039 struct tss_segment_16 tss_seg;
3041 u32 new_tss_base = get_desc_base(new_desc);
3043 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3044 if (ret != X86EMUL_CONTINUE)
3047 save_state_to_tss16(ctxt, &tss_seg);
3049 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3050 if (ret != X86EMUL_CONTINUE)
3053 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3054 if (ret != X86EMUL_CONTINUE)
3057 if (old_tss_sel != 0xffff) {
3058 tss_seg.prev_task_link = old_tss_sel;
3060 ret = linear_write_system(ctxt, new_tss_base,
3061 &tss_seg.prev_task_link,
3062 sizeof tss_seg.prev_task_link);
3063 if (ret != X86EMUL_CONTINUE)
3067 return load_state_from_tss16(ctxt, &tss_seg);
3070 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3071 struct tss_segment_32 *tss)
3073 /* CR3 and ldt selector are not saved intentionally */
3074 tss->eip = ctxt->_eip;
3075 tss->eflags = ctxt->eflags;
3076 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3077 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3078 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3079 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3080 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3081 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3082 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3083 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3085 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3086 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3087 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3088 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3089 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3090 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3093 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3094 struct tss_segment_32 *tss)
3099 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3100 return emulate_gp(ctxt, 0);
3101 ctxt->_eip = tss->eip;
3102 ctxt->eflags = tss->eflags | 2;
3104 /* General purpose registers */
3105 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3106 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3107 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3108 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3109 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3110 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3111 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3112 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3115 * SDM says that segment selectors are loaded before segment
3116 * descriptors. This is important because CPL checks will
3119 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3120 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3121 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3122 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3123 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3124 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3125 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3128 * If we're switching between Protected Mode and VM86, we need to make
3129 * sure to update the mode before loading the segment descriptors so
3130 * that the selectors are interpreted correctly.
3132 if (ctxt->eflags & X86_EFLAGS_VM) {
3133 ctxt->mode = X86EMUL_MODE_VM86;
3136 ctxt->mode = X86EMUL_MODE_PROT32;
3141 * Now load segment descriptors. If fault happenes at this stage
3142 * it is handled in a context of new task
3144 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3145 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3146 if (ret != X86EMUL_CONTINUE)
3148 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3149 X86_TRANSFER_TASK_SWITCH, NULL);
3150 if (ret != X86EMUL_CONTINUE)
3152 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3153 X86_TRANSFER_TASK_SWITCH, NULL);
3154 if (ret != X86EMUL_CONTINUE)
3156 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3157 X86_TRANSFER_TASK_SWITCH, NULL);
3158 if (ret != X86EMUL_CONTINUE)
3160 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3161 X86_TRANSFER_TASK_SWITCH, NULL);
3162 if (ret != X86EMUL_CONTINUE)
3164 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3165 X86_TRANSFER_TASK_SWITCH, NULL);
3166 if (ret != X86EMUL_CONTINUE)
3168 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3169 X86_TRANSFER_TASK_SWITCH, NULL);
3174 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3175 u16 tss_selector, u16 old_tss_sel,
3176 ulong old_tss_base, struct desc_struct *new_desc)
3178 struct tss_segment_32 tss_seg;
3180 u32 new_tss_base = get_desc_base(new_desc);
3181 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3182 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3184 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3185 if (ret != X86EMUL_CONTINUE)
3188 save_state_to_tss32(ctxt, &tss_seg);
3190 /* Only GP registers and segment selectors are saved */
3191 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3192 ldt_sel_offset - eip_offset);
3193 if (ret != X86EMUL_CONTINUE)
3196 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3197 if (ret != X86EMUL_CONTINUE)
3200 if (old_tss_sel != 0xffff) {
3201 tss_seg.prev_task_link = old_tss_sel;
3203 ret = linear_write_system(ctxt, new_tss_base,
3204 &tss_seg.prev_task_link,
3205 sizeof tss_seg.prev_task_link);
3206 if (ret != X86EMUL_CONTINUE)
3210 return load_state_from_tss32(ctxt, &tss_seg);
3213 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3214 u16 tss_selector, int idt_index, int reason,
3215 bool has_error_code, u32 error_code)
3217 const struct x86_emulate_ops *ops = ctxt->ops;
3218 struct desc_struct curr_tss_desc, next_tss_desc;
3220 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3221 ulong old_tss_base =
3222 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3224 ulong desc_addr, dr7;
3226 /* FIXME: old_tss_base == ~0 ? */
3228 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3229 if (ret != X86EMUL_CONTINUE)
3231 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3232 if (ret != X86EMUL_CONTINUE)
3235 /* FIXME: check that next_tss_desc is tss */
3238 * Check privileges. The three cases are task switch caused by...
3240 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3241 * 2. Exception/IRQ/iret: No check is performed
3242 * 3. jmp/call to TSS/task-gate: No check is performed since the
3243 * hardware checks it before exiting.
3245 if (reason == TASK_SWITCH_GATE) {
3246 if (idt_index != -1) {
3247 /* Software interrupts */
3248 struct desc_struct task_gate_desc;
3251 ret = read_interrupt_descriptor(ctxt, idt_index,
3253 if (ret != X86EMUL_CONTINUE)
3256 dpl = task_gate_desc.dpl;
3257 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3258 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3262 desc_limit = desc_limit_scaled(&next_tss_desc);
3263 if (!next_tss_desc.p ||
3264 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3265 desc_limit < 0x2b)) {
3266 return emulate_ts(ctxt, tss_selector & 0xfffc);
3269 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3270 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3271 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3274 if (reason == TASK_SWITCH_IRET)
3275 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3277 /* set back link to prev task only if NT bit is set in eflags
3278 note that old_tss_sel is not used after this point */
3279 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3280 old_tss_sel = 0xffff;
3282 if (next_tss_desc.type & 8)
3283 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3284 old_tss_base, &next_tss_desc);
3286 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3287 old_tss_base, &next_tss_desc);
3288 if (ret != X86EMUL_CONTINUE)
3291 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3292 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3294 if (reason != TASK_SWITCH_IRET) {
3295 next_tss_desc.type |= (1 << 1); /* set busy flag */
3296 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3299 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3300 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3302 if (has_error_code) {
3303 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3304 ctxt->lock_prefix = 0;
3305 ctxt->src.val = (unsigned long) error_code;
3306 ret = em_push(ctxt);
3309 ops->get_dr(ctxt, 7, &dr7);
3310 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3315 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3316 u16 tss_selector, int idt_index, int reason,
3317 bool has_error_code, u32 error_code)
3321 invalidate_registers(ctxt);
3322 ctxt->_eip = ctxt->eip;
3323 ctxt->dst.type = OP_NONE;
3325 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3326 has_error_code, error_code);
3328 if (rc == X86EMUL_CONTINUE) {
3329 ctxt->eip = ctxt->_eip;
3330 writeback_registers(ctxt);
3333 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3336 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3339 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3341 register_address_increment(ctxt, reg, df * op->bytes);
3342 op->addr.mem.ea = register_address(ctxt, reg);
3345 static int em_das(struct x86_emulate_ctxt *ctxt)
3348 bool af, cf, old_cf;
3350 cf = ctxt->eflags & X86_EFLAGS_CF;
3356 af = ctxt->eflags & X86_EFLAGS_AF;
3357 if ((al & 0x0f) > 9 || af) {
3359 cf = old_cf | (al >= 250);
3364 if (old_al > 0x99 || old_cf) {
3370 /* Set PF, ZF, SF */
3371 ctxt->src.type = OP_IMM;
3373 ctxt->src.bytes = 1;
3374 fastop(ctxt, em_or);
3375 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3377 ctxt->eflags |= X86_EFLAGS_CF;
3379 ctxt->eflags |= X86_EFLAGS_AF;
3380 return X86EMUL_CONTINUE;
3383 static int em_aam(struct x86_emulate_ctxt *ctxt)
3387 if (ctxt->src.val == 0)
3388 return emulate_de(ctxt);
3390 al = ctxt->dst.val & 0xff;
3391 ah = al / ctxt->src.val;
3392 al %= ctxt->src.val;
3394 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3396 /* Set PF, ZF, SF */
3397 ctxt->src.type = OP_IMM;
3399 ctxt->src.bytes = 1;
3400 fastop(ctxt, em_or);
3402 return X86EMUL_CONTINUE;
3405 static int em_aad(struct x86_emulate_ctxt *ctxt)
3407 u8 al = ctxt->dst.val & 0xff;
3408 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3410 al = (al + (ah * ctxt->src.val)) & 0xff;
3412 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3414 /* Set PF, ZF, SF */
3415 ctxt->src.type = OP_IMM;
3417 ctxt->src.bytes = 1;
3418 fastop(ctxt, em_or);
3420 return X86EMUL_CONTINUE;
3423 static int em_call(struct x86_emulate_ctxt *ctxt)
3426 long rel = ctxt->src.val;
3428 ctxt->src.val = (unsigned long)ctxt->_eip;
3429 rc = jmp_rel(ctxt, rel);
3430 if (rc != X86EMUL_CONTINUE)
3432 return em_push(ctxt);
3435 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3440 struct desc_struct old_desc, new_desc;
3441 const struct x86_emulate_ops *ops = ctxt->ops;
3442 int cpl = ctxt->ops->cpl(ctxt);
3443 enum x86emul_mode prev_mode = ctxt->mode;
3445 old_eip = ctxt->_eip;
3446 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3448 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3449 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3450 X86_TRANSFER_CALL_JMP, &new_desc);
3451 if (rc != X86EMUL_CONTINUE)
3454 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3455 if (rc != X86EMUL_CONTINUE)
3458 ctxt->src.val = old_cs;
3460 if (rc != X86EMUL_CONTINUE)
3463 ctxt->src.val = old_eip;
3465 /* If we failed, we tainted the memory, but the very least we should
3467 if (rc != X86EMUL_CONTINUE) {
3468 pr_warn_once("faulting far call emulation tainted memory\n");
3473 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3474 ctxt->mode = prev_mode;
3479 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3484 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3485 if (rc != X86EMUL_CONTINUE)
3487 rc = assign_eip_near(ctxt, eip);
3488 if (rc != X86EMUL_CONTINUE)
3490 rsp_increment(ctxt, ctxt->src.val);
3491 return X86EMUL_CONTINUE;
3494 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3496 /* Write back the register source. */
3497 ctxt->src.val = ctxt->dst.val;
3498 write_register_operand(&ctxt->src);
3500 /* Write back the memory destination with implicit LOCK prefix. */
3501 ctxt->dst.val = ctxt->src.orig_val;
3502 ctxt->lock_prefix = 1;
3503 return X86EMUL_CONTINUE;
3506 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3508 ctxt->dst.val = ctxt->src2.val;
3509 return fastop(ctxt, em_imul);
3512 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3514 ctxt->dst.type = OP_REG;
3515 ctxt->dst.bytes = ctxt->src.bytes;
3516 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3517 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3519 return X86EMUL_CONTINUE;
3522 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3526 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3527 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3528 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3529 return X86EMUL_CONTINUE;
3532 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3536 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3537 return emulate_gp(ctxt, 0);
3538 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3539 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3540 return X86EMUL_CONTINUE;
3543 static int em_mov(struct x86_emulate_ctxt *ctxt)
3545 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3546 return X86EMUL_CONTINUE;
3549 #define FFL(x) bit(X86_FEATURE_##x)
3551 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3553 u32 ebx, ecx, edx, eax = 1;
3557 * Check MOVBE is set in the guest-visible CPUID leaf.
3559 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3560 if (!(ecx & FFL(MOVBE)))
3561 return emulate_ud(ctxt);
3563 switch (ctxt->op_bytes) {
3566 * From MOVBE definition: "...When the operand size is 16 bits,
3567 * the upper word of the destination register remains unchanged
3570 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3571 * rules so we have to do the operation almost per hand.
3573 tmp = (u16)ctxt->src.val;
3574 ctxt->dst.val &= ~0xffffUL;
3575 ctxt->dst.val |= (unsigned long)swab16(tmp);
3578 ctxt->dst.val = swab32((u32)ctxt->src.val);
3581 ctxt->dst.val = swab64(ctxt->src.val);
3586 return X86EMUL_CONTINUE;
3589 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3591 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3592 return emulate_gp(ctxt, 0);
3594 /* Disable writeback. */
3595 ctxt->dst.type = OP_NONE;
3596 return X86EMUL_CONTINUE;
3599 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3603 if (ctxt->mode == X86EMUL_MODE_PROT64)
3604 val = ctxt->src.val & ~0ULL;
3606 val = ctxt->src.val & ~0U;
3608 /* #UD condition is already handled. */
3609 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3610 return emulate_gp(ctxt, 0);
3612 /* Disable writeback. */
3613 ctxt->dst.type = OP_NONE;
3614 return X86EMUL_CONTINUE;
3617 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3621 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3622 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3623 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3624 return emulate_gp(ctxt, 0);
3626 return X86EMUL_CONTINUE;
3629 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3633 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3634 return emulate_gp(ctxt, 0);
3636 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3637 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3638 return X86EMUL_CONTINUE;
3641 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3643 if (ctxt->modrm_reg > VCPU_SREG_GS)
3644 return emulate_ud(ctxt);
3646 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3647 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3648 ctxt->dst.bytes = 2;
3649 return X86EMUL_CONTINUE;
3652 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3654 u16 sel = ctxt->src.val;
3656 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3657 return emulate_ud(ctxt);
3659 if (ctxt->modrm_reg == VCPU_SREG_SS)
3660 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3662 /* Disable writeback. */
3663 ctxt->dst.type = OP_NONE;
3664 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3667 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3669 u16 sel = ctxt->src.val;
3671 /* Disable writeback. */
3672 ctxt->dst.type = OP_NONE;
3673 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3676 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3678 u16 sel = ctxt->src.val;
3680 /* Disable writeback. */
3681 ctxt->dst.type = OP_NONE;
3682 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3685 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3690 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3691 if (rc == X86EMUL_CONTINUE)
3692 ctxt->ops->invlpg(ctxt, linear);
3693 /* Disable writeback. */
3694 ctxt->dst.type = OP_NONE;
3695 return X86EMUL_CONTINUE;
3698 static int em_clts(struct x86_emulate_ctxt *ctxt)
3702 cr0 = ctxt->ops->get_cr(ctxt, 0);
3704 ctxt->ops->set_cr(ctxt, 0, cr0);
3705 return X86EMUL_CONTINUE;
3708 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3710 int rc = ctxt->ops->fix_hypercall(ctxt);
3712 if (rc != X86EMUL_CONTINUE)
3715 /* Let the processor re-execute the fixed hypercall */
3716 ctxt->_eip = ctxt->eip;
3717 /* Disable writeback. */
3718 ctxt->dst.type = OP_NONE;
3719 return X86EMUL_CONTINUE;
3722 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3723 void (*get)(struct x86_emulate_ctxt *ctxt,
3724 struct desc_ptr *ptr))
3726 struct desc_ptr desc_ptr;
3728 if (ctxt->mode == X86EMUL_MODE_PROT64)
3730 get(ctxt, &desc_ptr);
3731 if (ctxt->op_bytes == 2) {
3733 desc_ptr.address &= 0x00ffffff;
3735 /* Disable writeback. */
3736 ctxt->dst.type = OP_NONE;
3737 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3738 &desc_ptr, 2 + ctxt->op_bytes);
3741 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3743 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3746 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3748 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3751 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3753 struct desc_ptr desc_ptr;
3756 if (ctxt->mode == X86EMUL_MODE_PROT64)
3758 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3759 &desc_ptr.size, &desc_ptr.address,
3761 if (rc != X86EMUL_CONTINUE)
3763 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3764 is_noncanonical_address(desc_ptr.address))
3765 return emulate_gp(ctxt, 0);
3767 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3769 ctxt->ops->set_idt(ctxt, &desc_ptr);
3770 /* Disable writeback. */
3771 ctxt->dst.type = OP_NONE;
3772 return X86EMUL_CONTINUE;
3775 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3777 return em_lgdt_lidt(ctxt, true);
3780 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3782 return em_lgdt_lidt(ctxt, false);
3785 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3787 if (ctxt->dst.type == OP_MEM)
3788 ctxt->dst.bytes = 2;
3789 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3790 return X86EMUL_CONTINUE;
3793 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3795 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3796 | (ctxt->src.val & 0x0f));
3797 ctxt->dst.type = OP_NONE;
3798 return X86EMUL_CONTINUE;
3801 static int em_loop(struct x86_emulate_ctxt *ctxt)
3803 int rc = X86EMUL_CONTINUE;
3805 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3806 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3807 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3808 rc = jmp_rel(ctxt, ctxt->src.val);
3813 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3815 int rc = X86EMUL_CONTINUE;
3817 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3818 rc = jmp_rel(ctxt, ctxt->src.val);
3823 static int em_in(struct x86_emulate_ctxt *ctxt)
3825 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3827 return X86EMUL_IO_NEEDED;
3829 return X86EMUL_CONTINUE;
3832 static int em_out(struct x86_emulate_ctxt *ctxt)
3834 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3836 /* Disable writeback. */
3837 ctxt->dst.type = OP_NONE;
3838 return X86EMUL_CONTINUE;
3841 static int em_cli(struct x86_emulate_ctxt *ctxt)
3843 if (emulator_bad_iopl(ctxt))
3844 return emulate_gp(ctxt, 0);
3846 ctxt->eflags &= ~X86_EFLAGS_IF;
3847 return X86EMUL_CONTINUE;
3850 static int em_sti(struct x86_emulate_ctxt *ctxt)
3852 if (emulator_bad_iopl(ctxt))
3853 return emulate_gp(ctxt, 0);
3855 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3856 ctxt->eflags |= X86_EFLAGS_IF;
3857 return X86EMUL_CONTINUE;
3860 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3862 u32 eax, ebx, ecx, edx;
3864 eax = reg_read(ctxt, VCPU_REGS_RAX);
3865 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3866 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3867 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3868 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3869 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3870 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3871 return X86EMUL_CONTINUE;
3874 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3878 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3880 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3882 ctxt->eflags &= ~0xffUL;
3883 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3884 return X86EMUL_CONTINUE;
3887 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3889 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3890 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3891 return X86EMUL_CONTINUE;
3894 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3896 switch (ctxt->op_bytes) {
3897 #ifdef CONFIG_X86_64
3899 asm("bswap %0" : "+r"(ctxt->dst.val));
3903 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3906 return X86EMUL_CONTINUE;
3909 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3911 /* emulating clflush regardless of cpuid */
3912 return X86EMUL_CONTINUE;
3915 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3917 ctxt->dst.val = (s32) ctxt->src.val;
3918 return X86EMUL_CONTINUE;
3921 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3923 u32 eax = 1, ebx, ecx = 0, edx;
3925 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3926 if (!(edx & FFL(FXSR)))
3927 return emulate_ud(ctxt);
3929 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3930 return emulate_nm(ctxt);
3933 * Don't emulate a case that should never be hit, instead of working
3934 * around a lack of fxsave64/fxrstor64 on old compilers.
3936 if (ctxt->mode >= X86EMUL_MODE_PROT64)
3937 return X86EMUL_UNHANDLEABLE;
3939 return X86EMUL_CONTINUE;
3943 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3946 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3947 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3949 * 3) 64-bit mode with REX.W prefix
3950 * - like (2), but XMM 8-15 are being saved and restored
3951 * 4) 64-bit mode without REX.W prefix
3952 * - like (3), but FIP and FDP are 64 bit
3954 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3955 * desired result. (4) is not emulated.
3957 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3958 * and FPU DS) should match.
3960 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
3962 struct fxregs_state fx_state;
3966 rc = check_fxsr(ctxt);
3967 if (rc != X86EMUL_CONTINUE)
3970 ctxt->ops->get_fpu(ctxt);
3972 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
3974 ctxt->ops->put_fpu(ctxt);
3976 if (rc != X86EMUL_CONTINUE)
3979 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)
3980 size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]);
3982 size = offsetof(struct fxregs_state, xmm_space[0]);
3984 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
3987 static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt,
3988 struct fxregs_state *new)
3990 int rc = X86EMUL_CONTINUE;
3991 struct fxregs_state old;
3993 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old));
3994 if (rc != X86EMUL_CONTINUE)
3998 * 64 bit host will restore XMM 8-15, which is not correct on non-64
3999 * bit guests. Load the current values in order to preserve 64 bit
4000 * XMMs after fxrstor.
4002 #ifdef CONFIG_X86_64
4003 /* XXX: accessing XMM 8-15 very awkwardly */
4004 memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16);
4008 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but
4009 * does save and restore MXCSR.
4011 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))
4012 memcpy(new->xmm_space, old.xmm_space, 8 * 16);
4017 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4019 struct fxregs_state fx_state;
4022 rc = check_fxsr(ctxt);
4023 if (rc != X86EMUL_CONTINUE)
4026 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, 512);
4027 if (rc != X86EMUL_CONTINUE)
4030 if (fx_state.mxcsr >> 16)
4031 return emulate_gp(ctxt, 0);
4033 ctxt->ops->get_fpu(ctxt);
4035 if (ctxt->mode < X86EMUL_MODE_PROT64)
4036 rc = fxrstor_fixup(ctxt, &fx_state);
4038 if (rc == X86EMUL_CONTINUE)
4039 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4041 ctxt->ops->put_fpu(ctxt);
4046 static bool valid_cr(int nr)
4058 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4060 if (!valid_cr(ctxt->modrm_reg))
4061 return emulate_ud(ctxt);
4063 return X86EMUL_CONTINUE;
4066 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4068 u64 new_val = ctxt->src.val64;
4069 int cr = ctxt->modrm_reg;
4072 static u64 cr_reserved_bits[] = {
4073 0xffffffff00000000ULL,
4074 0, 0, 0, /* CR3 checked later */
4081 return emulate_ud(ctxt);
4083 if (new_val & cr_reserved_bits[cr])
4084 return emulate_gp(ctxt, 0);
4089 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4090 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4091 return emulate_gp(ctxt, 0);
4093 cr4 = ctxt->ops->get_cr(ctxt, 4);
4094 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4096 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4097 !(cr4 & X86_CR4_PAE))
4098 return emulate_gp(ctxt, 0);
4105 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4106 if (efer & EFER_LMA)
4107 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
4110 return emulate_gp(ctxt, 0);
4115 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4117 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4118 return emulate_gp(ctxt, 0);
4124 return X86EMUL_CONTINUE;
4127 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4131 ctxt->ops->get_dr(ctxt, 7, &dr7);
4133 /* Check if DR7.Global_Enable is set */
4134 return dr7 & (1 << 13);
4137 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4139 int dr = ctxt->modrm_reg;
4143 return emulate_ud(ctxt);
4145 cr4 = ctxt->ops->get_cr(ctxt, 4);
4146 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4147 return emulate_ud(ctxt);
4149 if (check_dr7_gd(ctxt)) {
4152 ctxt->ops->get_dr(ctxt, 6, &dr6);
4154 dr6 |= DR6_BD | DR6_RTM;
4155 ctxt->ops->set_dr(ctxt, 6, dr6);
4156 return emulate_db(ctxt);
4159 return X86EMUL_CONTINUE;
4162 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4164 u64 new_val = ctxt->src.val64;
4165 int dr = ctxt->modrm_reg;
4167 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4168 return emulate_gp(ctxt, 0);
4170 return check_dr_read(ctxt);
4173 static int check_svme(struct x86_emulate_ctxt *ctxt)
4177 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4179 if (!(efer & EFER_SVME))
4180 return emulate_ud(ctxt);
4182 return X86EMUL_CONTINUE;
4185 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4187 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4189 /* Valid physical address? */
4190 if (rax & 0xffff000000000000ULL)
4191 return emulate_gp(ctxt, 0);
4193 return check_svme(ctxt);
4196 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4198 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4200 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4201 return emulate_ud(ctxt);
4203 return X86EMUL_CONTINUE;
4206 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4208 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4209 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4211 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4212 ctxt->ops->check_pmc(ctxt, rcx))
4213 return emulate_gp(ctxt, 0);
4215 return X86EMUL_CONTINUE;
4218 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4220 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4221 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4222 return emulate_gp(ctxt, 0);
4224 return X86EMUL_CONTINUE;
4227 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4229 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4230 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4231 return emulate_gp(ctxt, 0);
4233 return X86EMUL_CONTINUE;
4236 #define D(_y) { .flags = (_y) }
4237 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4238 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4239 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4240 #define N D(NotImpl)
4241 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4242 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4243 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4244 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4245 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4246 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4247 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4248 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4249 #define II(_f, _e, _i) \
4250 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4251 #define IIP(_f, _e, _i, _p) \
4252 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4253 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4254 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4256 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4257 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4258 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4259 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4260 #define I2bvIP(_f, _e, _i, _p) \
4261 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4263 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4264 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4265 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4267 static const struct opcode group7_rm0[] = {
4269 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4273 static const struct opcode group7_rm1[] = {
4274 DI(SrcNone | Priv, monitor),
4275 DI(SrcNone | Priv, mwait),
4279 static const struct opcode group7_rm3[] = {
4280 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4281 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4282 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4283 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4284 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4285 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4286 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4287 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4290 static const struct opcode group7_rm7[] = {
4292 DIP(SrcNone, rdtscp, check_rdtsc),
4296 static const struct opcode group1[] = {
4298 F(Lock | PageTable, em_or),
4301 F(Lock | PageTable, em_and),
4307 static const struct opcode group1A[] = {
4308 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4311 static const struct opcode group2[] = {
4312 F(DstMem | ModRM, em_rol),
4313 F(DstMem | ModRM, em_ror),
4314 F(DstMem | ModRM, em_rcl),
4315 F(DstMem | ModRM, em_rcr),
4316 F(DstMem | ModRM, em_shl),
4317 F(DstMem | ModRM, em_shr),
4318 F(DstMem | ModRM, em_shl),
4319 F(DstMem | ModRM, em_sar),
4322 static const struct opcode group3[] = {
4323 F(DstMem | SrcImm | NoWrite, em_test),
4324 F(DstMem | SrcImm | NoWrite, em_test),
4325 F(DstMem | SrcNone | Lock, em_not),
4326 F(DstMem | SrcNone | Lock, em_neg),
4327 F(DstXacc | Src2Mem, em_mul_ex),
4328 F(DstXacc | Src2Mem, em_imul_ex),
4329 F(DstXacc | Src2Mem, em_div_ex),
4330 F(DstXacc | Src2Mem, em_idiv_ex),
4333 static const struct opcode group4[] = {
4334 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4335 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4339 static const struct opcode group5[] = {
4340 F(DstMem | SrcNone | Lock, em_inc),
4341 F(DstMem | SrcNone | Lock, em_dec),
4342 I(SrcMem | NearBranch, em_call_near_abs),
4343 I(SrcMemFAddr | ImplicitOps, em_call_far),
4344 I(SrcMem | NearBranch, em_jmp_abs),
4345 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4346 I(SrcMem | Stack, em_push), D(Undefined),
4349 static const struct opcode group6[] = {
4350 DI(Prot | DstMem, sldt),
4351 DI(Prot | DstMem, str),
4352 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4353 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4357 static const struct group_dual group7 = { {
4358 II(Mov | DstMem, em_sgdt, sgdt),
4359 II(Mov | DstMem, em_sidt, sidt),
4360 II(SrcMem | Priv, em_lgdt, lgdt),
4361 II(SrcMem | Priv, em_lidt, lidt),
4362 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4363 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4364 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4368 N, EXT(0, group7_rm3),
4369 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4370 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4374 static const struct opcode group8[] = {
4376 F(DstMem | SrcImmByte | NoWrite, em_bt),
4377 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4378 F(DstMem | SrcImmByte | Lock, em_btr),
4379 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4382 static const struct group_dual group9 = { {
4383 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4385 N, N, N, N, N, N, N, N,
4388 static const struct opcode group11[] = {
4389 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4393 static const struct gprefix pfx_0f_ae_7 = {
4394 I(SrcMem | ByteOp, em_clflush), N, N, N,
4397 static const struct group_dual group15 = { {
4398 I(ModRM | Aligned16, em_fxsave),
4399 I(ModRM | Aligned16, em_fxrstor),
4400 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4402 N, N, N, N, N, N, N, N,
4405 static const struct gprefix pfx_0f_6f_0f_7f = {
4406 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4409 static const struct instr_dual instr_dual_0f_2b = {
4413 static const struct gprefix pfx_0f_2b = {
4414 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4417 static const struct gprefix pfx_0f_28_0f_29 = {
4418 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4421 static const struct gprefix pfx_0f_e7 = {
4422 N, I(Sse, em_mov), N, N,
4425 static const struct escape escape_d9 = { {
4426 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4429 N, N, N, N, N, N, N, N,
4431 N, N, N, N, N, N, N, N,
4433 N, N, N, N, N, N, N, N,
4435 N, N, N, N, N, N, N, N,
4437 N, N, N, N, N, N, N, N,
4439 N, N, N, N, N, N, N, N,
4441 N, N, N, N, N, N, N, N,
4443 N, N, N, N, N, N, N, N,
4446 static const struct escape escape_db = { {
4447 N, N, N, N, N, N, N, N,
4450 N, N, N, N, N, N, N, N,
4452 N, N, N, N, N, N, N, N,
4454 N, N, N, N, N, N, N, N,
4456 N, N, N, N, N, N, N, N,
4458 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4460 N, N, N, N, N, N, N, N,
4462 N, N, N, N, N, N, N, N,
4464 N, N, N, N, N, N, N, N,
4467 static const struct escape escape_dd = { {
4468 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4471 N, N, N, N, N, N, N, N,
4473 N, N, N, N, N, N, N, N,
4475 N, N, N, N, N, N, N, N,
4477 N, N, N, N, N, N, N, N,
4479 N, N, N, N, N, N, N, N,
4481 N, N, N, N, N, N, N, N,
4483 N, N, N, N, N, N, N, N,
4485 N, N, N, N, N, N, N, N,
4488 static const struct instr_dual instr_dual_0f_c3 = {
4489 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4492 static const struct mode_dual mode_dual_63 = {
4493 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4496 static const struct opcode opcode_table[256] = {
4498 F6ALU(Lock, em_add),
4499 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4500 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4502 F6ALU(Lock | PageTable, em_or),
4503 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4506 F6ALU(Lock, em_adc),
4507 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4508 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4510 F6ALU(Lock, em_sbb),
4511 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4512 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4514 F6ALU(Lock | PageTable, em_and), N, N,
4516 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4518 F6ALU(Lock, em_xor), N, N,
4520 F6ALU(NoWrite, em_cmp), N, N,
4522 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4524 X8(I(SrcReg | Stack, em_push)),
4526 X8(I(DstReg | Stack, em_pop)),
4528 I(ImplicitOps | Stack | No64, em_pusha),
4529 I(ImplicitOps | Stack | No64, em_popa),
4530 N, MD(ModRM, &mode_dual_63),
4533 I(SrcImm | Mov | Stack, em_push),
4534 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4535 I(SrcImmByte | Mov | Stack, em_push),
4536 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4537 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4538 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4540 X16(D(SrcImmByte | NearBranch)),
4542 G(ByteOp | DstMem | SrcImm, group1),
4543 G(DstMem | SrcImm, group1),
4544 G(ByteOp | DstMem | SrcImm | No64, group1),
4545 G(DstMem | SrcImmByte, group1),
4546 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4547 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4549 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4550 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4551 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4552 D(ModRM | SrcMem | NoAccess | DstReg),
4553 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4556 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4558 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4559 I(SrcImmFAddr | No64, em_call_far), N,
4560 II(ImplicitOps | Stack, em_pushf, pushf),
4561 II(ImplicitOps | Stack, em_popf, popf),
4562 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4564 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4565 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4566 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4567 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4569 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4570 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4571 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4572 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4574 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4576 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4578 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4579 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4580 I(ImplicitOps | NearBranch, em_ret),
4581 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4582 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4583 G(ByteOp, group11), G(0, group11),
4585 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4586 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4587 I(ImplicitOps, em_ret_far),
4588 D(ImplicitOps), DI(SrcImmByte, intn),
4589 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4591 G(Src2One | ByteOp, group2), G(Src2One, group2),
4592 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4593 I(DstAcc | SrcImmUByte | No64, em_aam),
4594 I(DstAcc | SrcImmUByte | No64, em_aad),
4595 F(DstAcc | ByteOp | No64, em_salc),
4596 I(DstAcc | SrcXLat | ByteOp, em_mov),
4598 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4600 X3(I(SrcImmByte | NearBranch, em_loop)),
4601 I(SrcImmByte | NearBranch, em_jcxz),
4602 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4603 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4605 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4606 I(SrcImmFAddr | No64, em_jmp_far),
4607 D(SrcImmByte | ImplicitOps | NearBranch),
4608 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4609 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4611 N, DI(ImplicitOps, icebp), N, N,
4612 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4613 G(ByteOp, group3), G(0, group3),
4615 D(ImplicitOps), D(ImplicitOps),
4616 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4617 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4620 static const struct opcode twobyte_table[256] = {
4622 G(0, group6), GD(0, &group7), N, N,
4623 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4624 II(ImplicitOps | Priv, em_clts, clts), N,
4625 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4626 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4628 N, N, N, N, N, N, N, N,
4629 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4630 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4632 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4633 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4634 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4636 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4639 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4640 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4641 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4644 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4645 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4646 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4647 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4648 I(ImplicitOps | EmulateOnUD, em_sysenter),
4649 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4651 N, N, N, N, N, N, N, N,
4653 X16(D(DstReg | SrcMem | ModRM)),
4655 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4660 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4665 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4667 X16(D(SrcImm | NearBranch)),
4669 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4671 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4672 II(ImplicitOps, em_cpuid, cpuid),
4673 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4674 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4675 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4677 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4678 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4679 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4680 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4681 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4682 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4684 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4685 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4686 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4687 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4688 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4689 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4693 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4694 I(DstReg | SrcMem | ModRM, em_bsf_c),
4695 I(DstReg | SrcMem | ModRM, em_bsr_c),
4696 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4698 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4699 N, ID(0, &instr_dual_0f_c3),
4700 N, N, N, GD(0, &group9),
4702 X8(I(DstReg, em_bswap)),
4704 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4706 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4707 N, N, N, N, N, N, N, N,
4709 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4712 static const struct instr_dual instr_dual_0f_38_f0 = {
4713 I(DstReg | SrcMem | Mov, em_movbe), N
4716 static const struct instr_dual instr_dual_0f_38_f1 = {
4717 I(DstMem | SrcReg | Mov, em_movbe), N
4720 static const struct gprefix three_byte_0f_38_f0 = {
4721 ID(0, &instr_dual_0f_38_f0), N, N, N
4724 static const struct gprefix three_byte_0f_38_f1 = {
4725 ID(0, &instr_dual_0f_38_f1), N, N, N
4729 * Insns below are selected by the prefix which indexed by the third opcode
4732 static const struct opcode opcode_map_0f_38[256] = {
4734 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4736 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4738 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4739 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4760 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4764 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4770 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4771 unsigned size, bool sign_extension)
4773 int rc = X86EMUL_CONTINUE;
4777 op->addr.mem.ea = ctxt->_eip;
4778 /* NB. Immediates are sign-extended as necessary. */
4779 switch (op->bytes) {
4781 op->val = insn_fetch(s8, ctxt);
4784 op->val = insn_fetch(s16, ctxt);
4787 op->val = insn_fetch(s32, ctxt);
4790 op->val = insn_fetch(s64, ctxt);
4793 if (!sign_extension) {
4794 switch (op->bytes) {
4802 op->val &= 0xffffffff;
4810 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4813 int rc = X86EMUL_CONTINUE;
4817 decode_register_operand(ctxt, op);
4820 rc = decode_imm(ctxt, op, 1, false);
4823 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4827 if (ctxt->d & BitOp)
4828 fetch_bit_operand(ctxt);
4829 op->orig_val = op->val;
4832 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4836 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4837 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4838 fetch_register_operand(op);
4839 op->orig_val = op->val;
4843 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4844 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4845 fetch_register_operand(op);
4846 op->orig_val = op->val;
4849 if (ctxt->d & ByteOp) {
4854 op->bytes = ctxt->op_bytes;
4855 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4856 fetch_register_operand(op);
4857 op->orig_val = op->val;
4861 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4863 register_address(ctxt, VCPU_REGS_RDI);
4864 op->addr.mem.seg = VCPU_SREG_ES;
4871 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4872 fetch_register_operand(op);
4877 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4880 rc = decode_imm(ctxt, op, 1, true);
4888 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4891 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4894 ctxt->memop.bytes = 1;
4895 if (ctxt->memop.type == OP_REG) {
4896 ctxt->memop.addr.reg = decode_register(ctxt,
4897 ctxt->modrm_rm, true);
4898 fetch_register_operand(&ctxt->memop);
4902 ctxt->memop.bytes = 2;
4905 ctxt->memop.bytes = 4;
4908 rc = decode_imm(ctxt, op, 2, false);
4911 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4915 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4917 register_address(ctxt, VCPU_REGS_RSI);
4918 op->addr.mem.seg = ctxt->seg_override;
4924 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4927 reg_read(ctxt, VCPU_REGS_RBX) +
4928 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4929 op->addr.mem.seg = ctxt->seg_override;
4934 op->addr.mem.ea = ctxt->_eip;
4935 op->bytes = ctxt->op_bytes + 2;
4936 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4939 ctxt->memop.bytes = ctxt->op_bytes + 2;
4943 op->val = VCPU_SREG_ES;
4947 op->val = VCPU_SREG_CS;
4951 op->val = VCPU_SREG_SS;
4955 op->val = VCPU_SREG_DS;
4959 op->val = VCPU_SREG_FS;
4963 op->val = VCPU_SREG_GS;
4966 /* Special instructions do their own operand decoding. */
4968 op->type = OP_NONE; /* Disable writeback. */
4976 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4978 int rc = X86EMUL_CONTINUE;
4979 int mode = ctxt->mode;
4980 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4981 bool op_prefix = false;
4982 bool has_seg_override = false;
4983 struct opcode opcode;
4985 struct desc_struct desc;
4987 ctxt->memop.type = OP_NONE;
4988 ctxt->memopp = NULL;
4989 ctxt->_eip = ctxt->eip;
4990 ctxt->fetch.ptr = ctxt->fetch.data;
4991 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4992 ctxt->opcode_len = 1;
4994 memcpy(ctxt->fetch.data, insn, insn_len);
4996 rc = __do_insn_fetch_bytes(ctxt, 1);
4997 if (rc != X86EMUL_CONTINUE)
5002 case X86EMUL_MODE_REAL:
5003 case X86EMUL_MODE_VM86:
5004 def_op_bytes = def_ad_bytes = 2;
5005 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5007 def_op_bytes = def_ad_bytes = 4;
5009 case X86EMUL_MODE_PROT16:
5010 def_op_bytes = def_ad_bytes = 2;
5012 case X86EMUL_MODE_PROT32:
5013 def_op_bytes = def_ad_bytes = 4;
5015 #ifdef CONFIG_X86_64
5016 case X86EMUL_MODE_PROT64:
5022 return EMULATION_FAILED;
5025 ctxt->op_bytes = def_op_bytes;
5026 ctxt->ad_bytes = def_ad_bytes;
5028 /* Legacy prefixes. */
5030 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5031 case 0x66: /* operand-size override */
5033 /* switch between 2/4 bytes */
5034 ctxt->op_bytes = def_op_bytes ^ 6;
5036 case 0x67: /* address-size override */
5037 if (mode == X86EMUL_MODE_PROT64)
5038 /* switch between 4/8 bytes */
5039 ctxt->ad_bytes = def_ad_bytes ^ 12;
5041 /* switch between 2/4 bytes */
5042 ctxt->ad_bytes = def_ad_bytes ^ 6;
5044 case 0x26: /* ES override */
5045 has_seg_override = true;
5046 ctxt->seg_override = VCPU_SREG_ES;
5048 case 0x2e: /* CS override */
5049 has_seg_override = true;
5050 ctxt->seg_override = VCPU_SREG_CS;
5052 case 0x36: /* SS override */
5053 has_seg_override = true;
5054 ctxt->seg_override = VCPU_SREG_SS;
5056 case 0x3e: /* DS override */
5057 has_seg_override = true;
5058 ctxt->seg_override = VCPU_SREG_DS;
5060 case 0x64: /* FS override */
5061 has_seg_override = true;
5062 ctxt->seg_override = VCPU_SREG_FS;
5064 case 0x65: /* GS override */
5065 has_seg_override = true;
5066 ctxt->seg_override = VCPU_SREG_GS;
5068 case 0x40 ... 0x4f: /* REX */
5069 if (mode != X86EMUL_MODE_PROT64)
5071 ctxt->rex_prefix = ctxt->b;
5073 case 0xf0: /* LOCK */
5074 ctxt->lock_prefix = 1;
5076 case 0xf2: /* REPNE/REPNZ */
5077 case 0xf3: /* REP/REPE/REPZ */
5078 ctxt->rep_prefix = ctxt->b;
5084 /* Any legacy prefix after a REX prefix nullifies its effect. */
5086 ctxt->rex_prefix = 0;
5092 if (ctxt->rex_prefix & 8)
5093 ctxt->op_bytes = 8; /* REX.W */
5095 /* Opcode byte(s). */
5096 opcode = opcode_table[ctxt->b];
5097 /* Two-byte opcode? */
5098 if (ctxt->b == 0x0f) {
5099 ctxt->opcode_len = 2;
5100 ctxt->b = insn_fetch(u8, ctxt);
5101 opcode = twobyte_table[ctxt->b];
5103 /* 0F_38 opcode map */
5104 if (ctxt->b == 0x38) {
5105 ctxt->opcode_len = 3;
5106 ctxt->b = insn_fetch(u8, ctxt);
5107 opcode = opcode_map_0f_38[ctxt->b];
5110 ctxt->d = opcode.flags;
5112 if (ctxt->d & ModRM)
5113 ctxt->modrm = insn_fetch(u8, ctxt);
5115 /* vex-prefix instructions are not implemented */
5116 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5117 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5121 while (ctxt->d & GroupMask) {
5122 switch (ctxt->d & GroupMask) {
5124 goffset = (ctxt->modrm >> 3) & 7;
5125 opcode = opcode.u.group[goffset];
5128 goffset = (ctxt->modrm >> 3) & 7;
5129 if ((ctxt->modrm >> 6) == 3)
5130 opcode = opcode.u.gdual->mod3[goffset];
5132 opcode = opcode.u.gdual->mod012[goffset];
5135 goffset = ctxt->modrm & 7;
5136 opcode = opcode.u.group[goffset];
5139 if (ctxt->rep_prefix && op_prefix)
5140 return EMULATION_FAILED;
5141 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5142 switch (simd_prefix) {
5143 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5144 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5145 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5146 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5150 if (ctxt->modrm > 0xbf) {
5151 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5152 u32 index = array_index_nospec(
5153 ctxt->modrm - 0xc0, size);
5155 opcode = opcode.u.esc->high[index];
5157 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5161 if ((ctxt->modrm >> 6) == 3)
5162 opcode = opcode.u.idual->mod3;
5164 opcode = opcode.u.idual->mod012;
5167 if (ctxt->mode == X86EMUL_MODE_PROT64)
5168 opcode = opcode.u.mdual->mode64;
5170 opcode = opcode.u.mdual->mode32;
5173 return EMULATION_FAILED;
5176 ctxt->d &= ~(u64)GroupMask;
5177 ctxt->d |= opcode.flags;
5182 return EMULATION_FAILED;
5184 ctxt->execute = opcode.u.execute;
5186 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5187 return EMULATION_FAILED;
5189 if (unlikely(ctxt->d &
5190 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5193 * These are copied unconditionally here, and checked unconditionally
5194 * in x86_emulate_insn.
5196 ctxt->check_perm = opcode.check_perm;
5197 ctxt->intercept = opcode.intercept;
5199 if (ctxt->d & NotImpl)
5200 return EMULATION_FAILED;
5202 if (mode == X86EMUL_MODE_PROT64) {
5203 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5205 else if (ctxt->d & NearBranch)
5209 if (ctxt->d & Op3264) {
5210 if (mode == X86EMUL_MODE_PROT64)
5216 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5220 ctxt->op_bytes = 16;
5221 else if (ctxt->d & Mmx)
5225 /* ModRM and SIB bytes. */
5226 if (ctxt->d & ModRM) {
5227 rc = decode_modrm(ctxt, &ctxt->memop);
5228 if (!has_seg_override) {
5229 has_seg_override = true;
5230 ctxt->seg_override = ctxt->modrm_seg;
5232 } else if (ctxt->d & MemAbs)
5233 rc = decode_abs(ctxt, &ctxt->memop);
5234 if (rc != X86EMUL_CONTINUE)
5237 if (!has_seg_override)
5238 ctxt->seg_override = VCPU_SREG_DS;
5240 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5243 * Decode and fetch the source operand: register, memory
5246 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5247 if (rc != X86EMUL_CONTINUE)
5251 * Decode and fetch the second source operand: register, memory
5254 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5255 if (rc != X86EMUL_CONTINUE)
5258 /* Decode and fetch the destination operand: register or memory. */
5259 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5261 if (ctxt->rip_relative && likely(ctxt->memopp))
5262 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5263 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5266 if (rc == X86EMUL_PROPAGATE_FAULT)
5267 ctxt->have_exception = true;
5268 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5271 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5273 return ctxt->d & PageTable;
5276 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5278 /* The second termination condition only applies for REPE
5279 * and REPNE. Test if the repeat string operation prefix is
5280 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5281 * corresponding termination condition according to:
5282 * - if REPE/REPZ and ZF = 0 then done
5283 * - if REPNE/REPNZ and ZF = 1 then done
5285 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5286 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5287 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5288 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5289 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5290 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5296 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5300 ctxt->ops->get_fpu(ctxt);
5301 rc = asm_safe("fwait");
5302 ctxt->ops->put_fpu(ctxt);
5304 if (unlikely(rc != X86EMUL_CONTINUE))
5305 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5307 return X86EMUL_CONTINUE;
5310 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5313 if (op->type == OP_MM)
5314 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5317 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5319 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5320 if (!(ctxt->d & ByteOp))
5321 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5322 asm("push %[flags]; popf; " CALL_NOSPEC "; pushf; pop %[flags]\n"
5323 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5324 [thunk_target]"+S"(fop)
5325 : "c"(ctxt->src2.val));
5326 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5327 if (!fop) /* exception is returned in fop variable */
5328 return emulate_de(ctxt);
5329 return X86EMUL_CONTINUE;
5332 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5334 memset(&ctxt->rip_relative, 0,
5335 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5337 ctxt->io_read.pos = 0;
5338 ctxt->io_read.end = 0;
5339 ctxt->mem_read.end = 0;
5342 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5344 const struct x86_emulate_ops *ops = ctxt->ops;
5345 int rc = X86EMUL_CONTINUE;
5346 int saved_dst_type = ctxt->dst.type;
5347 unsigned emul_flags;
5349 ctxt->mem_read.pos = 0;
5351 /* LOCK prefix is allowed only with some instructions */
5352 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5353 rc = emulate_ud(ctxt);
5357 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5358 rc = emulate_ud(ctxt);
5362 emul_flags = ctxt->ops->get_hflags(ctxt);
5363 if (unlikely(ctxt->d &
5364 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5365 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5366 (ctxt->d & Undefined)) {
5367 rc = emulate_ud(ctxt);
5371 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5372 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5373 rc = emulate_ud(ctxt);
5377 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5378 rc = emulate_nm(ctxt);
5382 if (ctxt->d & Mmx) {
5383 rc = flush_pending_x87_faults(ctxt);
5384 if (rc != X86EMUL_CONTINUE)
5387 * Now that we know the fpu is exception safe, we can fetch
5390 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5391 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5392 if (!(ctxt->d & Mov))
5393 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5396 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5397 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5398 X86_ICPT_PRE_EXCEPT);
5399 if (rc != X86EMUL_CONTINUE)
5403 /* Instruction can only be executed in protected mode */
5404 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5405 rc = emulate_ud(ctxt);
5409 /* Privileged instruction can be executed only in CPL=0 */
5410 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5411 if (ctxt->d & PrivUD)
5412 rc = emulate_ud(ctxt);
5414 rc = emulate_gp(ctxt, 0);
5418 /* Do instruction specific permission checks */
5419 if (ctxt->d & CheckPerm) {
5420 rc = ctxt->check_perm(ctxt);
5421 if (rc != X86EMUL_CONTINUE)
5425 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5426 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5427 X86_ICPT_POST_EXCEPT);
5428 if (rc != X86EMUL_CONTINUE)
5432 if (ctxt->rep_prefix && (ctxt->d & String)) {
5433 /* All REP prefixes have the same first termination condition */
5434 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5435 string_registers_quirk(ctxt);
5436 ctxt->eip = ctxt->_eip;
5437 ctxt->eflags &= ~X86_EFLAGS_RF;
5443 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5444 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5445 ctxt->src.valptr, ctxt->src.bytes);
5446 if (rc != X86EMUL_CONTINUE)
5448 ctxt->src.orig_val64 = ctxt->src.val64;
5451 if (ctxt->src2.type == OP_MEM) {
5452 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5453 &ctxt->src2.val, ctxt->src2.bytes);
5454 if (rc != X86EMUL_CONTINUE)
5458 if ((ctxt->d & DstMask) == ImplicitOps)
5462 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5463 /* optimisation - avoid slow emulated read if Mov */
5464 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5465 &ctxt->dst.val, ctxt->dst.bytes);
5466 if (rc != X86EMUL_CONTINUE) {
5467 if (!(ctxt->d & NoWrite) &&
5468 rc == X86EMUL_PROPAGATE_FAULT &&
5469 ctxt->exception.vector == PF_VECTOR)
5470 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5474 /* Copy full 64-bit value for CMPXCHG8B. */
5475 ctxt->dst.orig_val64 = ctxt->dst.val64;
5479 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5480 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5481 X86_ICPT_POST_MEMACCESS);
5482 if (rc != X86EMUL_CONTINUE)
5486 if (ctxt->rep_prefix && (ctxt->d & String))
5487 ctxt->eflags |= X86_EFLAGS_RF;
5489 ctxt->eflags &= ~X86_EFLAGS_RF;
5491 if (ctxt->execute) {
5492 if (ctxt->d & Fastop) {
5493 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5494 rc = fastop(ctxt, fop);
5495 if (rc != X86EMUL_CONTINUE)
5499 rc = ctxt->execute(ctxt);
5500 if (rc != X86EMUL_CONTINUE)
5505 if (ctxt->opcode_len == 2)
5507 else if (ctxt->opcode_len == 3)
5508 goto threebyte_insn;
5511 case 0x70 ... 0x7f: /* jcc (short) */
5512 if (test_cc(ctxt->b, ctxt->eflags))
5513 rc = jmp_rel(ctxt, ctxt->src.val);
5515 case 0x8d: /* lea r16/r32, m */
5516 ctxt->dst.val = ctxt->src.addr.mem.ea;
5518 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5519 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5520 ctxt->dst.type = OP_NONE;
5524 case 0x98: /* cbw/cwde/cdqe */
5525 switch (ctxt->op_bytes) {
5526 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5527 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5528 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5531 case 0xcc: /* int3 */
5532 rc = emulate_int(ctxt, 3);
5534 case 0xcd: /* int n */
5535 rc = emulate_int(ctxt, ctxt->src.val);
5537 case 0xce: /* into */
5538 if (ctxt->eflags & X86_EFLAGS_OF)
5539 rc = emulate_int(ctxt, 4);
5541 case 0xe9: /* jmp rel */
5542 case 0xeb: /* jmp rel short */
5543 rc = jmp_rel(ctxt, ctxt->src.val);
5544 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5546 case 0xf4: /* hlt */
5547 ctxt->ops->halt(ctxt);
5549 case 0xf5: /* cmc */
5550 /* complement carry flag from eflags reg */
5551 ctxt->eflags ^= X86_EFLAGS_CF;
5553 case 0xf8: /* clc */
5554 ctxt->eflags &= ~X86_EFLAGS_CF;
5556 case 0xf9: /* stc */
5557 ctxt->eflags |= X86_EFLAGS_CF;
5559 case 0xfc: /* cld */
5560 ctxt->eflags &= ~X86_EFLAGS_DF;
5562 case 0xfd: /* std */
5563 ctxt->eflags |= X86_EFLAGS_DF;
5566 goto cannot_emulate;
5569 if (rc != X86EMUL_CONTINUE)
5573 if (ctxt->d & SrcWrite) {
5574 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5575 rc = writeback(ctxt, &ctxt->src);
5576 if (rc != X86EMUL_CONTINUE)
5579 if (!(ctxt->d & NoWrite)) {
5580 rc = writeback(ctxt, &ctxt->dst);
5581 if (rc != X86EMUL_CONTINUE)
5586 * restore dst type in case the decoding will be reused
5587 * (happens for string instruction )
5589 ctxt->dst.type = saved_dst_type;
5591 if ((ctxt->d & SrcMask) == SrcSI)
5592 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5594 if ((ctxt->d & DstMask) == DstDI)
5595 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5597 if (ctxt->rep_prefix && (ctxt->d & String)) {
5599 struct read_cache *r = &ctxt->io_read;
5600 if ((ctxt->d & SrcMask) == SrcSI)
5601 count = ctxt->src.count;
5603 count = ctxt->dst.count;
5604 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5606 if (!string_insn_completed(ctxt)) {
5608 * Re-enter guest when pio read ahead buffer is empty
5609 * or, if it is not used, after each 1024 iteration.
5611 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5612 (r->end == 0 || r->end != r->pos)) {
5614 * Reset read cache. Usually happens before
5615 * decode, but since instruction is restarted
5616 * we have to do it here.
5618 ctxt->mem_read.end = 0;
5619 writeback_registers(ctxt);
5620 return EMULATION_RESTART;
5622 goto done; /* skip rip writeback */
5624 ctxt->eflags &= ~X86_EFLAGS_RF;
5627 ctxt->eip = ctxt->_eip;
5630 if (rc == X86EMUL_PROPAGATE_FAULT) {
5631 WARN_ON(ctxt->exception.vector > 0x1f);
5632 ctxt->have_exception = true;
5634 if (rc == X86EMUL_INTERCEPTED)
5635 return EMULATION_INTERCEPTED;
5637 if (rc == X86EMUL_CONTINUE)
5638 writeback_registers(ctxt);
5640 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5644 case 0x09: /* wbinvd */
5645 (ctxt->ops->wbinvd)(ctxt);
5647 case 0x08: /* invd */
5648 case 0x0d: /* GrpP (prefetch) */
5649 case 0x18: /* Grp16 (prefetch/nop) */
5650 case 0x1f: /* nop */
5652 case 0x20: /* mov cr, reg */
5653 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5655 case 0x21: /* mov from dr to reg */
5656 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5658 case 0x40 ... 0x4f: /* cmov */
5659 if (test_cc(ctxt->b, ctxt->eflags))
5660 ctxt->dst.val = ctxt->src.val;
5661 else if (ctxt->op_bytes != 4)
5662 ctxt->dst.type = OP_NONE; /* no writeback */
5664 case 0x80 ... 0x8f: /* jnz rel, etc*/
5665 if (test_cc(ctxt->b, ctxt->eflags))
5666 rc = jmp_rel(ctxt, ctxt->src.val);
5668 case 0x90 ... 0x9f: /* setcc r/m8 */
5669 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5671 case 0xb6 ... 0xb7: /* movzx */
5672 ctxt->dst.bytes = ctxt->op_bytes;
5673 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5674 : (u16) ctxt->src.val;
5676 case 0xbe ... 0xbf: /* movsx */
5677 ctxt->dst.bytes = ctxt->op_bytes;
5678 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5679 (s16) ctxt->src.val;
5682 goto cannot_emulate;
5687 if (rc != X86EMUL_CONTINUE)
5693 return EMULATION_FAILED;
5696 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5698 invalidate_registers(ctxt);
5701 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5703 writeback_registers(ctxt);