1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
79 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
80 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
82 /* Source 2 operand type */
83 #define Src2None (0<<29)
84 #define Src2CL (1<<29)
85 #define Src2ImmByte (2<<29)
86 #define Src2One (3<<29)
87 #define Src2Imm16 (4<<29)
88 #define Src2Mask (7<<29)
91 Group1_80, Group1_81, Group1_82, Group1_83,
92 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
96 static u32 opcode_table[256] = {
98 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
99 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
100 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
101 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
103 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
104 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
105 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
106 ImplicitOps | Stack | No64, 0,
108 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
109 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
110 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
111 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
113 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
114 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
115 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
116 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
118 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
119 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
120 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
122 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
123 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
126 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
127 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
131 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
132 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
135 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
137 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
139 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
140 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
142 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
143 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
145 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
146 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
149 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
150 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
151 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
153 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
156 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
159 Group | Group1_80, Group | Group1_81,
160 Group | Group1_82, Group | Group1_83,
161 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
162 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
164 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
165 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
166 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
167 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
169 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
171 0, 0, SrcImm | Src2Imm16 | No64, 0,
172 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
174 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
175 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
176 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
177 ByteOp | ImplicitOps | String, ImplicitOps | String,
179 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
180 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
181 ByteOp | ImplicitOps | String, ImplicitOps | String,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
184 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
185 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
186 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
189 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
190 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
191 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
193 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
194 0, ImplicitOps | Stack, 0, 0,
195 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
197 0, 0, 0, ImplicitOps | Stack,
198 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
200 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
201 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
204 0, 0, 0, 0, 0, 0, 0, 0,
207 ByteOp | SrcImmUByte, SrcImmUByte,
208 ByteOp | SrcImmUByte, SrcImmUByte,
210 SrcImm | Stack, SrcImm | ImplicitOps,
211 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
212 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
213 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
216 ImplicitOps | Priv, ImplicitOps, Group | Group3_Byte, Group | Group3,
218 ImplicitOps, 0, ImplicitOps, ImplicitOps,
219 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
222 static u32 twobyte_table[256] = {
224 0, Group | GroupDual | Group7, 0, 0,
225 0, ImplicitOps, ImplicitOps | Priv, 0,
226 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
227 0, ImplicitOps | ModRM, 0, 0,
229 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
231 ModRM | ImplicitOps | Priv, ModRM | Priv,
232 ModRM | ImplicitOps | Priv, ModRM | Priv,
234 0, 0, 0, 0, 0, 0, 0, 0,
236 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
237 ImplicitOps, ImplicitOps | Priv, 0, 0,
238 0, 0, 0, 0, 0, 0, 0, 0,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
243 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
245 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
246 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
247 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
248 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
250 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
252 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
254 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
256 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
257 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
259 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
261 ImplicitOps | Stack, ImplicitOps | Stack,
262 0, DstMem | SrcReg | ModRM | BitOp,
263 DstMem | SrcReg | Src2ImmByte | ModRM,
264 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
266 ImplicitOps | Stack, ImplicitOps | Stack,
267 0, DstMem | SrcReg | ModRM | BitOp | Lock,
268 DstMem | SrcReg | Src2ImmByte | ModRM,
269 DstMem | SrcReg | Src2CL | ModRM,
272 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
273 0, DstMem | SrcReg | ModRM | BitOp | Lock,
274 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
275 DstReg | SrcMem16 | ModRM | Mov,
278 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
279 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
280 DstReg | SrcMem16 | ModRM | Mov,
282 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
283 0, 0, 0, Group | GroupDual | Group9,
284 0, 0, 0, 0, 0, 0, 0, 0,
286 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
288 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
290 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
293 static u32 group_table[] = {
295 ByteOp | DstMem | SrcImm | ModRM | Lock,
296 ByteOp | DstMem | SrcImm | ModRM | Lock,
297 ByteOp | DstMem | SrcImm | ModRM | Lock,
298 ByteOp | DstMem | SrcImm | ModRM | Lock,
299 ByteOp | DstMem | SrcImm | ModRM | Lock,
300 ByteOp | DstMem | SrcImm | ModRM | Lock,
301 ByteOp | DstMem | SrcImm | ModRM | Lock,
302 ByteOp | DstMem | SrcImm | ModRM,
304 DstMem | SrcImm | ModRM | Lock,
305 DstMem | SrcImm | ModRM | Lock,
306 DstMem | SrcImm | ModRM | Lock,
307 DstMem | SrcImm | ModRM | Lock,
308 DstMem | SrcImm | ModRM | Lock,
309 DstMem | SrcImm | ModRM | Lock,
310 DstMem | SrcImm | ModRM | Lock,
311 DstMem | SrcImm | ModRM,
313 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
314 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
315 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
316 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
317 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
318 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
319 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
320 ByteOp | DstMem | SrcImm | ModRM | No64,
322 DstMem | SrcImmByte | ModRM | Lock,
323 DstMem | SrcImmByte | ModRM | Lock,
324 DstMem | SrcImmByte | ModRM | Lock,
325 DstMem | SrcImmByte | ModRM | Lock,
326 DstMem | SrcImmByte | ModRM | Lock,
327 DstMem | SrcImmByte | ModRM | Lock,
328 DstMem | SrcImmByte | ModRM | Lock,
329 DstMem | SrcImmByte | ModRM,
331 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
333 ByteOp | SrcImm | DstMem | ModRM, 0,
334 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
337 DstMem | SrcImm | ModRM, 0,
338 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
341 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
344 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
345 SrcMem | ModRM | Stack, 0,
346 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
348 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
349 SrcNone | ModRM | DstMem | Mov, 0,
350 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
353 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
354 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
356 0, ImplicitOps | ModRM | Lock, 0, 0, 0, 0, 0, 0,
359 static u32 group2_table[] = {
361 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM,
362 SrcNone | ModRM | DstMem | Mov, 0,
363 SrcMem16 | ModRM | Mov, 0,
365 0, 0, 0, 0, 0, 0, 0, 0,
368 /* EFLAGS bit definitions. */
369 #define EFLG_ID (1<<21)
370 #define EFLG_VIP (1<<20)
371 #define EFLG_VIF (1<<19)
372 #define EFLG_AC (1<<18)
373 #define EFLG_VM (1<<17)
374 #define EFLG_RF (1<<16)
375 #define EFLG_IOPL (3<<12)
376 #define EFLG_NT (1<<14)
377 #define EFLG_OF (1<<11)
378 #define EFLG_DF (1<<10)
379 #define EFLG_IF (1<<9)
380 #define EFLG_TF (1<<8)
381 #define EFLG_SF (1<<7)
382 #define EFLG_ZF (1<<6)
383 #define EFLG_AF (1<<4)
384 #define EFLG_PF (1<<2)
385 #define EFLG_CF (1<<0)
388 * Instruction emulation:
389 * Most instructions are emulated directly via a fragment of inline assembly
390 * code. This allows us to save/restore EFLAGS and thus very easily pick up
391 * any modified flags.
394 #if defined(CONFIG_X86_64)
395 #define _LO32 "k" /* force 32-bit operand */
396 #define _STK "%%rsp" /* stack pointer */
397 #elif defined(__i386__)
398 #define _LO32 "" /* force 32-bit operand */
399 #define _STK "%%esp" /* stack pointer */
403 * These EFLAGS bits are restored from saved value during emulation, and
404 * any changes are written back to the saved value after emulation.
406 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
408 /* Before executing instruction: restore necessary bits in EFLAGS. */
409 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
410 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
411 "movl %"_sav",%"_LO32 _tmp"; " \
414 "movl %"_msk",%"_LO32 _tmp"; " \
415 "andl %"_LO32 _tmp",("_STK"); " \
417 "notl %"_LO32 _tmp"; " \
418 "andl %"_LO32 _tmp",("_STK"); " \
419 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
421 "orl %"_LO32 _tmp",("_STK"); " \
425 /* After executing instruction: write-back necessary bits in EFLAGS. */
426 #define _POST_EFLAGS(_sav, _msk, _tmp) \
427 /* _sav |= EFLAGS & _msk; */ \
430 "andl %"_msk",%"_LO32 _tmp"; " \
431 "orl %"_LO32 _tmp",%"_sav"; "
439 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
441 __asm__ __volatile__ ( \
442 _PRE_EFLAGS("0", "4", "2") \
443 _op _suffix " %"_x"3,%1; " \
444 _POST_EFLAGS("0", "4", "2") \
445 : "=m" (_eflags), "=m" ((_dst).val), \
447 : _y ((_src).val), "i" (EFLAGS_MASK)); \
451 /* Raw emulation: instruction has two explicit operands. */
452 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
454 unsigned long _tmp; \
456 switch ((_dst).bytes) { \
458 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
461 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
464 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
469 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
471 unsigned long _tmp; \
472 switch ((_dst).bytes) { \
474 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
477 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
478 _wx, _wy, _lx, _ly, _qx, _qy); \
483 /* Source operand is byte-sized and may be restricted to just %cl. */
484 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
485 __emulate_2op(_op, _src, _dst, _eflags, \
486 "b", "c", "b", "c", "b", "c", "b", "c")
488 /* Source operand is byte, word, long or quad sized. */
489 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
490 __emulate_2op(_op, _src, _dst, _eflags, \
491 "b", "q", "w", "r", _LO32, "r", "", "r")
493 /* Source operand is word, long or quad sized. */
494 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
495 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
496 "w", "r", _LO32, "r", "", "r")
498 /* Instruction has three operands and one operand is stored in ECX register */
499 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
501 unsigned long _tmp; \
502 _type _clv = (_cl).val; \
503 _type _srcv = (_src).val; \
504 _type _dstv = (_dst).val; \
506 __asm__ __volatile__ ( \
507 _PRE_EFLAGS("0", "5", "2") \
508 _op _suffix " %4,%1 \n" \
509 _POST_EFLAGS("0", "5", "2") \
510 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
511 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
514 (_cl).val = (unsigned long) _clv; \
515 (_src).val = (unsigned long) _srcv; \
516 (_dst).val = (unsigned long) _dstv; \
519 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
521 switch ((_dst).bytes) { \
523 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
524 "w", unsigned short); \
527 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
528 "l", unsigned int); \
531 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
532 "q", unsigned long)); \
537 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
539 unsigned long _tmp; \
541 __asm__ __volatile__ ( \
542 _PRE_EFLAGS("0", "3", "2") \
543 _op _suffix " %1; " \
544 _POST_EFLAGS("0", "3", "2") \
545 : "=m" (_eflags), "+m" ((_dst).val), \
547 : "i" (EFLAGS_MASK)); \
550 /* Instruction has only one explicit operand (no source operand). */
551 #define emulate_1op(_op, _dst, _eflags) \
553 switch ((_dst).bytes) { \
554 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
555 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
556 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
557 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
561 /* Fetch next part of the instruction being emulated. */
562 #define insn_fetch(_type, _size, _eip) \
563 ({ unsigned long _x; \
564 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
571 static inline unsigned long ad_mask(struct decode_cache *c)
573 return (1UL << (c->ad_bytes << 3)) - 1;
576 /* Access/update address held in a register, based on addressing mode. */
577 static inline unsigned long
578 address_mask(struct decode_cache *c, unsigned long reg)
580 if (c->ad_bytes == sizeof(unsigned long))
583 return reg & ad_mask(c);
586 static inline unsigned long
587 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
589 return base + address_mask(c, reg);
593 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
595 if (c->ad_bytes == sizeof(unsigned long))
598 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
601 static inline void jmp_rel(struct decode_cache *c, int rel)
603 register_address_increment(c, &c->eip, rel);
606 static void set_seg_override(struct decode_cache *c, int seg)
608 c->has_seg_override = true;
609 c->seg_override = seg;
612 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
614 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
617 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
620 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
621 struct decode_cache *c)
623 if (!c->has_seg_override)
626 return seg_base(ctxt, c->seg_override);
629 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
631 return seg_base(ctxt, VCPU_SREG_ES);
634 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
636 return seg_base(ctxt, VCPU_SREG_SS);
639 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
640 struct x86_emulate_ops *ops,
641 unsigned long linear, u8 *dest)
643 struct fetch_cache *fc = &ctxt->decode.fetch;
647 if (linear < fc->start || linear >= fc->end) {
648 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
649 rc = ops->fetch(linear, fc->data, size, ctxt->vcpu, NULL);
650 if (rc != X86EMUL_CONTINUE)
653 fc->end = linear + size;
655 *dest = fc->data[linear - fc->start];
656 return X86EMUL_CONTINUE;
659 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
660 struct x86_emulate_ops *ops,
661 unsigned long eip, void *dest, unsigned size)
665 /* x86 instructions are limited to 15 bytes. */
666 if (eip + size - ctxt->decode.eip_orig > 15)
667 return X86EMUL_UNHANDLEABLE;
668 eip += ctxt->cs_base;
670 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
671 if (rc != X86EMUL_CONTINUE)
674 return X86EMUL_CONTINUE;
678 * Given the 'reg' portion of a ModRM byte, and a register block, return a
679 * pointer into the block that addresses the relevant register.
680 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
682 static void *decode_register(u8 modrm_reg, unsigned long *regs,
687 p = ®s[modrm_reg];
688 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
689 p = (unsigned char *)®s[modrm_reg & 3] + 1;
693 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
694 struct x86_emulate_ops *ops,
696 u16 *size, unsigned long *address, int op_bytes)
703 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
705 if (rc != X86EMUL_CONTINUE)
707 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
712 static int test_cc(unsigned int condition, unsigned int flags)
716 switch ((condition & 15) >> 1) {
718 rc |= (flags & EFLG_OF);
720 case 1: /* b/c/nae */
721 rc |= (flags & EFLG_CF);
724 rc |= (flags & EFLG_ZF);
727 rc |= (flags & (EFLG_CF|EFLG_ZF));
730 rc |= (flags & EFLG_SF);
733 rc |= (flags & EFLG_PF);
736 rc |= (flags & EFLG_ZF);
739 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
743 /* Odd condition identifiers (lsb == 1) have inverted sense. */
744 return (!!rc ^ (condition & 1));
747 static void decode_register_operand(struct operand *op,
748 struct decode_cache *c,
751 unsigned reg = c->modrm_reg;
752 int highbyte_regs = c->rex_prefix == 0;
755 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
757 if ((c->d & ByteOp) && !inhibit_bytereg) {
758 op->ptr = decode_register(reg, c->regs, highbyte_regs);
759 op->val = *(u8 *)op->ptr;
762 op->ptr = decode_register(reg, c->regs, 0);
763 op->bytes = c->op_bytes;
766 op->val = *(u16 *)op->ptr;
769 op->val = *(u32 *)op->ptr;
772 op->val = *(u64 *) op->ptr;
776 op->orig_val = op->val;
779 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
780 struct x86_emulate_ops *ops)
782 struct decode_cache *c = &ctxt->decode;
784 int index_reg = 0, base_reg = 0, scale;
785 int rc = X86EMUL_CONTINUE;
788 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
789 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
790 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
793 c->modrm = insn_fetch(u8, 1, c->eip);
794 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
795 c->modrm_reg |= (c->modrm & 0x38) >> 3;
796 c->modrm_rm |= (c->modrm & 0x07);
800 if (c->modrm_mod == 3) {
801 c->modrm_ptr = decode_register(c->modrm_rm,
802 c->regs, c->d & ByteOp);
803 c->modrm_val = *(unsigned long *)c->modrm_ptr;
807 if (c->ad_bytes == 2) {
808 unsigned bx = c->regs[VCPU_REGS_RBX];
809 unsigned bp = c->regs[VCPU_REGS_RBP];
810 unsigned si = c->regs[VCPU_REGS_RSI];
811 unsigned di = c->regs[VCPU_REGS_RDI];
813 /* 16-bit ModR/M decode. */
814 switch (c->modrm_mod) {
816 if (c->modrm_rm == 6)
817 c->modrm_ea += insn_fetch(u16, 2, c->eip);
820 c->modrm_ea += insn_fetch(s8, 1, c->eip);
823 c->modrm_ea += insn_fetch(u16, 2, c->eip);
826 switch (c->modrm_rm) {
828 c->modrm_ea += bx + si;
831 c->modrm_ea += bx + di;
834 c->modrm_ea += bp + si;
837 c->modrm_ea += bp + di;
846 if (c->modrm_mod != 0)
853 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
854 (c->modrm_rm == 6 && c->modrm_mod != 0))
855 if (!c->has_seg_override)
856 set_seg_override(c, VCPU_SREG_SS);
857 c->modrm_ea = (u16)c->modrm_ea;
859 /* 32/64-bit ModR/M decode. */
860 if ((c->modrm_rm & 7) == 4) {
861 sib = insn_fetch(u8, 1, c->eip);
862 index_reg |= (sib >> 3) & 7;
866 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
867 c->modrm_ea += insn_fetch(s32, 4, c->eip);
869 c->modrm_ea += c->regs[base_reg];
871 c->modrm_ea += c->regs[index_reg] << scale;
872 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
873 if (ctxt->mode == X86EMUL_MODE_PROT64)
876 c->modrm_ea += c->regs[c->modrm_rm];
877 switch (c->modrm_mod) {
879 if (c->modrm_rm == 5)
880 c->modrm_ea += insn_fetch(s32, 4, c->eip);
883 c->modrm_ea += insn_fetch(s8, 1, c->eip);
886 c->modrm_ea += insn_fetch(s32, 4, c->eip);
894 static int decode_abs(struct x86_emulate_ctxt *ctxt,
895 struct x86_emulate_ops *ops)
897 struct decode_cache *c = &ctxt->decode;
898 int rc = X86EMUL_CONTINUE;
900 switch (c->ad_bytes) {
902 c->modrm_ea = insn_fetch(u16, 2, c->eip);
905 c->modrm_ea = insn_fetch(u32, 4, c->eip);
908 c->modrm_ea = insn_fetch(u64, 8, c->eip);
916 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
918 struct decode_cache *c = &ctxt->decode;
919 int rc = X86EMUL_CONTINUE;
920 int mode = ctxt->mode;
921 int def_op_bytes, def_ad_bytes, group;
923 /* Shadow copy of register state. Committed on successful emulation. */
925 memset(c, 0, sizeof(struct decode_cache));
926 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
927 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
928 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
931 case X86EMUL_MODE_REAL:
932 case X86EMUL_MODE_VM86:
933 case X86EMUL_MODE_PROT16:
934 def_op_bytes = def_ad_bytes = 2;
936 case X86EMUL_MODE_PROT32:
937 def_op_bytes = def_ad_bytes = 4;
940 case X86EMUL_MODE_PROT64:
949 c->op_bytes = def_op_bytes;
950 c->ad_bytes = def_ad_bytes;
952 /* Legacy prefixes. */
954 switch (c->b = insn_fetch(u8, 1, c->eip)) {
955 case 0x66: /* operand-size override */
956 /* switch between 2/4 bytes */
957 c->op_bytes = def_op_bytes ^ 6;
959 case 0x67: /* address-size override */
960 if (mode == X86EMUL_MODE_PROT64)
961 /* switch between 4/8 bytes */
962 c->ad_bytes = def_ad_bytes ^ 12;
964 /* switch between 2/4 bytes */
965 c->ad_bytes = def_ad_bytes ^ 6;
967 case 0x26: /* ES override */
968 case 0x2e: /* CS override */
969 case 0x36: /* SS override */
970 case 0x3e: /* DS override */
971 set_seg_override(c, (c->b >> 3) & 3);
973 case 0x64: /* FS override */
974 case 0x65: /* GS override */
975 set_seg_override(c, c->b & 7);
977 case 0x40 ... 0x4f: /* REX */
978 if (mode != X86EMUL_MODE_PROT64)
980 c->rex_prefix = c->b;
982 case 0xf0: /* LOCK */
985 case 0xf2: /* REPNE/REPNZ */
986 c->rep_prefix = REPNE_PREFIX;
988 case 0xf3: /* REP/REPE/REPZ */
989 c->rep_prefix = REPE_PREFIX;
995 /* Any legacy prefix after a REX prefix nullifies its effect. */
1004 if (c->rex_prefix & 8)
1005 c->op_bytes = 8; /* REX.W */
1007 /* Opcode byte(s). */
1008 c->d = opcode_table[c->b];
1010 /* Two-byte opcode? */
1013 c->b = insn_fetch(u8, 1, c->eip);
1014 c->d = twobyte_table[c->b];
1019 group = c->d & GroupMask;
1020 c->modrm = insn_fetch(u8, 1, c->eip);
1023 group = (group << 3) + ((c->modrm >> 3) & 7);
1024 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1025 c->d = group2_table[group];
1027 c->d = group_table[group];
1032 DPRINTF("Cannot emulate %02x\n", c->b);
1036 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1039 /* ModRM and SIB bytes. */
1041 rc = decode_modrm(ctxt, ops);
1042 else if (c->d & MemAbs)
1043 rc = decode_abs(ctxt, ops);
1044 if (rc != X86EMUL_CONTINUE)
1047 if (!c->has_seg_override)
1048 set_seg_override(c, VCPU_SREG_DS);
1050 if (!(!c->twobyte && c->b == 0x8d))
1051 c->modrm_ea += seg_override_base(ctxt, c);
1053 if (c->ad_bytes != 8)
1054 c->modrm_ea = (u32)c->modrm_ea;
1056 * Decode and fetch the source operand: register, memory
1059 switch (c->d & SrcMask) {
1063 decode_register_operand(&c->src, c, 0);
1072 c->src.bytes = (c->d & ByteOp) ? 1 :
1074 /* Don't fetch the address for invlpg: it could be unmapped. */
1075 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1079 * For instructions with a ModR/M byte, switch to register
1080 * access if Mod = 3.
1082 if ((c->d & ModRM) && c->modrm_mod == 3) {
1083 c->src.type = OP_REG;
1084 c->src.val = c->modrm_val;
1085 c->src.ptr = c->modrm_ptr;
1088 c->src.type = OP_MEM;
1092 c->src.type = OP_IMM;
1093 c->src.ptr = (unsigned long *)c->eip;
1094 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1095 if (c->src.bytes == 8)
1097 /* NB. Immediates are sign-extended as necessary. */
1098 switch (c->src.bytes) {
1100 c->src.val = insn_fetch(s8, 1, c->eip);
1103 c->src.val = insn_fetch(s16, 2, c->eip);
1106 c->src.val = insn_fetch(s32, 4, c->eip);
1109 if ((c->d & SrcMask) == SrcImmU) {
1110 switch (c->src.bytes) {
1115 c->src.val &= 0xffff;
1118 c->src.val &= 0xffffffff;
1125 c->src.type = OP_IMM;
1126 c->src.ptr = (unsigned long *)c->eip;
1128 if ((c->d & SrcMask) == SrcImmByte)
1129 c->src.val = insn_fetch(s8, 1, c->eip);
1131 c->src.val = insn_fetch(u8, 1, c->eip);
1140 * Decode and fetch the second source operand: register, memory
1143 switch (c->d & Src2Mask) {
1148 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1151 c->src2.type = OP_IMM;
1152 c->src2.ptr = (unsigned long *)c->eip;
1154 c->src2.val = insn_fetch(u8, 1, c->eip);
1157 c->src2.type = OP_IMM;
1158 c->src2.ptr = (unsigned long *)c->eip;
1160 c->src2.val = insn_fetch(u16, 2, c->eip);
1168 /* Decode and fetch the destination operand: register or memory. */
1169 switch (c->d & DstMask) {
1171 /* Special instructions do their own operand decoding. */
1174 decode_register_operand(&c->dst, c,
1175 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1178 if ((c->d & ModRM) && c->modrm_mod == 3) {
1179 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1180 c->dst.type = OP_REG;
1181 c->dst.val = c->dst.orig_val = c->modrm_val;
1182 c->dst.ptr = c->modrm_ptr;
1185 c->dst.type = OP_MEM;
1188 c->dst.type = OP_REG;
1189 c->dst.bytes = c->op_bytes;
1190 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1191 switch (c->op_bytes) {
1193 c->dst.val = *(u8 *)c->dst.ptr;
1196 c->dst.val = *(u16 *)c->dst.ptr;
1199 c->dst.val = *(u32 *)c->dst.ptr;
1202 c->dst.orig_val = c->dst.val;
1206 if (c->rip_relative)
1207 c->modrm_ea += c->eip;
1210 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1213 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1215 struct decode_cache *c = &ctxt->decode;
1217 c->dst.type = OP_MEM;
1218 c->dst.bytes = c->op_bytes;
1219 c->dst.val = c->src.val;
1220 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1221 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1222 c->regs[VCPU_REGS_RSP]);
1225 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1226 struct x86_emulate_ops *ops,
1227 void *dest, int len)
1229 struct decode_cache *c = &ctxt->decode;
1232 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1233 c->regs[VCPU_REGS_RSP]),
1234 dest, len, ctxt->vcpu);
1235 if (rc != X86EMUL_CONTINUE)
1238 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1242 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1243 struct x86_emulate_ops *ops,
1244 void *dest, int len)
1247 unsigned long val, change_mask;
1248 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1249 int cpl = kvm_x86_ops->get_cpl(ctxt->vcpu);
1251 rc = emulate_pop(ctxt, ops, &val, len);
1252 if (rc != X86EMUL_CONTINUE)
1255 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1256 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1258 switch(ctxt->mode) {
1259 case X86EMUL_MODE_PROT64:
1260 case X86EMUL_MODE_PROT32:
1261 case X86EMUL_MODE_PROT16:
1263 change_mask |= EFLG_IOPL;
1265 change_mask |= EFLG_IF;
1267 case X86EMUL_MODE_VM86:
1269 kvm_inject_gp(ctxt->vcpu, 0);
1270 return X86EMUL_PROPAGATE_FAULT;
1272 change_mask |= EFLG_IF;
1274 default: /* real mode */
1275 change_mask |= (EFLG_IOPL | EFLG_IF);
1279 *(unsigned long *)dest =
1280 (ctxt->eflags & ~change_mask) | (val & change_mask);
1285 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1287 struct decode_cache *c = &ctxt->decode;
1288 struct kvm_segment segment;
1290 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1292 c->src.val = segment.selector;
1296 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1297 struct x86_emulate_ops *ops, int seg)
1299 struct decode_cache *c = &ctxt->decode;
1300 unsigned long selector;
1303 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1304 if (rc != X86EMUL_CONTINUE)
1307 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, seg);
1311 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1313 struct decode_cache *c = &ctxt->decode;
1314 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1315 int reg = VCPU_REGS_RAX;
1317 while (reg <= VCPU_REGS_RDI) {
1318 (reg == VCPU_REGS_RSP) ?
1319 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1326 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1327 struct x86_emulate_ops *ops)
1329 struct decode_cache *c = &ctxt->decode;
1330 int rc = X86EMUL_CONTINUE;
1331 int reg = VCPU_REGS_RDI;
1333 while (reg >= VCPU_REGS_RAX) {
1334 if (reg == VCPU_REGS_RSP) {
1335 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1340 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1341 if (rc != X86EMUL_CONTINUE)
1348 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1349 struct x86_emulate_ops *ops)
1351 struct decode_cache *c = &ctxt->decode;
1353 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1356 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1358 struct decode_cache *c = &ctxt->decode;
1359 switch (c->modrm_reg) {
1361 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1364 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1367 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1370 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1372 case 4: /* sal/shl */
1373 case 6: /* sal/shl */
1374 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1377 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1380 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1385 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1386 struct x86_emulate_ops *ops)
1388 struct decode_cache *c = &ctxt->decode;
1389 int rc = X86EMUL_CONTINUE;
1391 switch (c->modrm_reg) {
1392 case 0 ... 1: /* test */
1393 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1396 c->dst.val = ~c->dst.val;
1399 emulate_1op("neg", c->dst, ctxt->eflags);
1402 DPRINTF("Cannot emulate %02x\n", c->b);
1403 rc = X86EMUL_UNHANDLEABLE;
1409 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1410 struct x86_emulate_ops *ops)
1412 struct decode_cache *c = &ctxt->decode;
1414 switch (c->modrm_reg) {
1416 emulate_1op("inc", c->dst, ctxt->eflags);
1419 emulate_1op("dec", c->dst, ctxt->eflags);
1421 case 2: /* call near abs */ {
1424 c->eip = c->src.val;
1425 c->src.val = old_eip;
1429 case 4: /* jmp abs */
1430 c->eip = c->src.val;
1436 return X86EMUL_CONTINUE;
1439 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1440 struct x86_emulate_ops *ops,
1441 unsigned long memop)
1443 struct decode_cache *c = &ctxt->decode;
1447 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1448 if (rc != X86EMUL_CONTINUE)
1451 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1452 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1454 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1455 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1456 ctxt->eflags &= ~EFLG_ZF;
1459 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1460 (u32) c->regs[VCPU_REGS_RBX];
1462 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1463 if (rc != X86EMUL_CONTINUE)
1465 ctxt->eflags |= EFLG_ZF;
1467 return X86EMUL_CONTINUE;
1470 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1471 struct x86_emulate_ops *ops)
1473 struct decode_cache *c = &ctxt->decode;
1477 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1478 if (rc != X86EMUL_CONTINUE)
1480 if (c->op_bytes == 4)
1481 c->eip = (u32)c->eip;
1482 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1483 if (rc != X86EMUL_CONTINUE)
1485 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, VCPU_SREG_CS);
1489 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1490 struct x86_emulate_ops *ops)
1493 struct decode_cache *c = &ctxt->decode;
1495 switch (c->dst.type) {
1497 /* The 4-byte case *is* correct:
1498 * in 64-bit mode we zero-extend.
1500 switch (c->dst.bytes) {
1502 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1505 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1508 *c->dst.ptr = (u32)c->dst.val;
1509 break; /* 64b: zero-ext */
1511 *c->dst.ptr = c->dst.val;
1517 rc = ops->cmpxchg_emulated(
1518 (unsigned long)c->dst.ptr,
1524 rc = ops->write_emulated(
1525 (unsigned long)c->dst.ptr,
1529 if (rc != X86EMUL_CONTINUE)
1538 return X86EMUL_CONTINUE;
1541 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1543 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1545 * an sti; sti; sequence only disable interrupts for the first
1546 * instruction. So, if the last instruction, be it emulated or
1547 * not, left the system with the INT_STI flag enabled, it
1548 * means that the last instruction is an sti. We should not
1549 * leave the flag on in this case. The same goes for mov ss
1551 if (!(int_shadow & mask))
1552 ctxt->interruptibility = mask;
1556 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1557 struct kvm_segment *cs, struct kvm_segment *ss)
1559 memset(cs, 0, sizeof(struct kvm_segment));
1560 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1561 memset(ss, 0, sizeof(struct kvm_segment));
1563 cs->l = 0; /* will be adjusted later */
1564 cs->base = 0; /* flat segment */
1565 cs->g = 1; /* 4kb granularity */
1566 cs->limit = 0xffffffff; /* 4GB limit */
1567 cs->type = 0x0b; /* Read, Execute, Accessed */
1569 cs->dpl = 0; /* will be adjusted later */
1574 ss->base = 0; /* flat segment */
1575 ss->limit = 0xffffffff; /* 4GB limit */
1576 ss->g = 1; /* 4kb granularity */
1578 ss->type = 0x03; /* Read/Write, Accessed */
1579 ss->db = 1; /* 32bit stack segment */
1585 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1587 struct decode_cache *c = &ctxt->decode;
1588 struct kvm_segment cs, ss;
1591 /* syscall is not available in real mode */
1592 if (ctxt->mode == X86EMUL_MODE_REAL || ctxt->mode == X86EMUL_MODE_VM86)
1593 return X86EMUL_UNHANDLEABLE;
1595 setup_syscalls_segments(ctxt, &cs, &ss);
1597 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1599 cs.selector = (u16)(msr_data & 0xfffc);
1600 ss.selector = (u16)(msr_data + 8);
1602 if (is_long_mode(ctxt->vcpu)) {
1606 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1607 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1609 c->regs[VCPU_REGS_RCX] = c->eip;
1610 if (is_long_mode(ctxt->vcpu)) {
1611 #ifdef CONFIG_X86_64
1612 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1614 kvm_x86_ops->get_msr(ctxt->vcpu,
1615 ctxt->mode == X86EMUL_MODE_PROT64 ?
1616 MSR_LSTAR : MSR_CSTAR, &msr_data);
1619 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1620 ctxt->eflags &= ~(msr_data | EFLG_RF);
1624 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1625 c->eip = (u32)msr_data;
1627 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1630 return X86EMUL_CONTINUE;
1634 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1636 struct decode_cache *c = &ctxt->decode;
1637 struct kvm_segment cs, ss;
1640 /* inject #GP if in real mode */
1641 if (ctxt->mode == X86EMUL_MODE_REAL) {
1642 kvm_inject_gp(ctxt->vcpu, 0);
1643 return X86EMUL_UNHANDLEABLE;
1646 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1647 * Therefore, we inject an #UD.
1649 if (ctxt->mode == X86EMUL_MODE_PROT64)
1650 return X86EMUL_UNHANDLEABLE;
1652 setup_syscalls_segments(ctxt, &cs, &ss);
1654 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1655 switch (ctxt->mode) {
1656 case X86EMUL_MODE_PROT32:
1657 if ((msr_data & 0xfffc) == 0x0) {
1658 kvm_inject_gp(ctxt->vcpu, 0);
1659 return X86EMUL_PROPAGATE_FAULT;
1662 case X86EMUL_MODE_PROT64:
1663 if (msr_data == 0x0) {
1664 kvm_inject_gp(ctxt->vcpu, 0);
1665 return X86EMUL_PROPAGATE_FAULT;
1670 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1671 cs.selector = (u16)msr_data;
1672 cs.selector &= ~SELECTOR_RPL_MASK;
1673 ss.selector = cs.selector + 8;
1674 ss.selector &= ~SELECTOR_RPL_MASK;
1675 if (ctxt->mode == X86EMUL_MODE_PROT64
1676 || is_long_mode(ctxt->vcpu)) {
1681 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1682 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1684 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1687 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1688 c->regs[VCPU_REGS_RSP] = msr_data;
1690 return X86EMUL_CONTINUE;
1694 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1696 struct decode_cache *c = &ctxt->decode;
1697 struct kvm_segment cs, ss;
1701 /* inject #GP if in real mode or Virtual 8086 mode */
1702 if (ctxt->mode == X86EMUL_MODE_REAL ||
1703 ctxt->mode == X86EMUL_MODE_VM86) {
1704 kvm_inject_gp(ctxt->vcpu, 0);
1705 return X86EMUL_UNHANDLEABLE;
1708 setup_syscalls_segments(ctxt, &cs, &ss);
1710 if ((c->rex_prefix & 0x8) != 0x0)
1711 usermode = X86EMUL_MODE_PROT64;
1713 usermode = X86EMUL_MODE_PROT32;
1717 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1719 case X86EMUL_MODE_PROT32:
1720 cs.selector = (u16)(msr_data + 16);
1721 if ((msr_data & 0xfffc) == 0x0) {
1722 kvm_inject_gp(ctxt->vcpu, 0);
1723 return X86EMUL_PROPAGATE_FAULT;
1725 ss.selector = (u16)(msr_data + 24);
1727 case X86EMUL_MODE_PROT64:
1728 cs.selector = (u16)(msr_data + 32);
1729 if (msr_data == 0x0) {
1730 kvm_inject_gp(ctxt->vcpu, 0);
1731 return X86EMUL_PROPAGATE_FAULT;
1733 ss.selector = cs.selector + 8;
1738 cs.selector |= SELECTOR_RPL_MASK;
1739 ss.selector |= SELECTOR_RPL_MASK;
1741 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1742 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1744 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1745 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1747 return X86EMUL_CONTINUE;
1750 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
1753 if (ctxt->mode == X86EMUL_MODE_REAL)
1755 if (ctxt->mode == X86EMUL_MODE_VM86)
1757 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1758 return kvm_x86_ops->get_cpl(ctxt->vcpu) > iopl;
1761 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1762 struct x86_emulate_ops *ops,
1765 struct kvm_segment tr_seg;
1768 u8 perm, bit_idx = port & 0x7;
1769 unsigned mask = (1 << len) - 1;
1771 kvm_get_segment(ctxt->vcpu, &tr_seg, VCPU_SREG_TR);
1772 if (tr_seg.unusable)
1774 if (tr_seg.limit < 103)
1776 r = ops->read_std(tr_seg.base + 102, &io_bitmap_ptr, 2, ctxt->vcpu,
1778 if (r != X86EMUL_CONTINUE)
1780 if (io_bitmap_ptr + port/8 > tr_seg.limit)
1782 r = ops->read_std(tr_seg.base + io_bitmap_ptr + port/8, &perm, 1,
1784 if (r != X86EMUL_CONTINUE)
1786 if ((perm >> bit_idx) & mask)
1791 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1792 struct x86_emulate_ops *ops,
1795 if (emulator_bad_iopl(ctxt))
1796 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1802 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1804 unsigned long memop = 0;
1806 unsigned long saved_eip = 0;
1807 struct decode_cache *c = &ctxt->decode;
1810 int rc = X86EMUL_CONTINUE;
1812 ctxt->interruptibility = 0;
1814 /* Shadow copy of register state. Committed on successful emulation.
1815 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1819 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1822 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
1823 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
1827 /* LOCK prefix is allowed only with some instructions */
1828 if (c->lock_prefix && !(c->d & Lock)) {
1829 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
1833 /* Privileged instruction can be executed only in CPL=0 */
1834 if ((c->d & Priv) && kvm_x86_ops->get_cpl(ctxt->vcpu)) {
1835 kvm_inject_gp(ctxt->vcpu, 0);
1839 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1840 memop = c->modrm_ea;
1842 if (c->rep_prefix && (c->d & String)) {
1843 /* All REP prefixes have the same first termination condition */
1844 if (c->regs[VCPU_REGS_RCX] == 0) {
1845 kvm_rip_write(ctxt->vcpu, c->eip);
1848 /* The second termination condition only applies for REPE
1849 * and REPNE. Test if the repeat string operation prefix is
1850 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1851 * corresponding termination condition according to:
1852 * - if REPE/REPZ and ZF = 0 then done
1853 * - if REPNE/REPNZ and ZF = 1 then done
1855 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1856 (c->b == 0xae) || (c->b == 0xaf)) {
1857 if ((c->rep_prefix == REPE_PREFIX) &&
1858 ((ctxt->eflags & EFLG_ZF) == 0)) {
1859 kvm_rip_write(ctxt->vcpu, c->eip);
1862 if ((c->rep_prefix == REPNE_PREFIX) &&
1863 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1864 kvm_rip_write(ctxt->vcpu, c->eip);
1868 c->regs[VCPU_REGS_RCX]--;
1869 c->eip = kvm_rip_read(ctxt->vcpu);
1872 if (c->src.type == OP_MEM) {
1873 c->src.ptr = (unsigned long *)memop;
1875 rc = ops->read_emulated((unsigned long)c->src.ptr,
1879 if (rc != X86EMUL_CONTINUE)
1881 c->src.orig_val = c->src.val;
1884 if ((c->d & DstMask) == ImplicitOps)
1888 if (c->dst.type == OP_MEM) {
1889 c->dst.ptr = (unsigned long *)memop;
1890 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1893 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1895 c->dst.ptr = (void *)c->dst.ptr +
1896 (c->src.val & mask) / 8;
1898 if (!(c->d & Mov)) {
1899 /* optimisation - avoid slow emulated read */
1900 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1904 if (rc != X86EMUL_CONTINUE)
1908 c->dst.orig_val = c->dst.val;
1918 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1920 case 0x06: /* push es */
1921 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1923 case 0x07: /* pop es */
1924 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1925 if (rc != X86EMUL_CONTINUE)
1930 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1932 case 0x0e: /* push cs */
1933 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1937 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1939 case 0x16: /* push ss */
1940 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1942 case 0x17: /* pop ss */
1943 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1944 if (rc != X86EMUL_CONTINUE)
1949 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1951 case 0x1e: /* push ds */
1952 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1954 case 0x1f: /* pop ds */
1955 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1956 if (rc != X86EMUL_CONTINUE)
1961 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1965 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1969 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1973 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1975 case 0x40 ... 0x47: /* inc r16/r32 */
1976 emulate_1op("inc", c->dst, ctxt->eflags);
1978 case 0x48 ... 0x4f: /* dec r16/r32 */
1979 emulate_1op("dec", c->dst, ctxt->eflags);
1981 case 0x50 ... 0x57: /* push reg */
1984 case 0x58 ... 0x5f: /* pop reg */
1986 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1987 if (rc != X86EMUL_CONTINUE)
1990 case 0x60: /* pusha */
1991 emulate_pusha(ctxt);
1993 case 0x61: /* popa */
1994 rc = emulate_popa(ctxt, ops);
1995 if (rc != X86EMUL_CONTINUE)
1998 case 0x63: /* movsxd */
1999 if (ctxt->mode != X86EMUL_MODE_PROT64)
2000 goto cannot_emulate;
2001 c->dst.val = (s32) c->src.val;
2003 case 0x68: /* push imm */
2004 case 0x6a: /* push imm8 */
2007 case 0x6c: /* insb */
2008 case 0x6d: /* insw/insd */
2009 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2010 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2011 kvm_inject_gp(ctxt->vcpu, 0);
2014 if (kvm_emulate_pio_string(ctxt->vcpu,
2016 (c->d & ByteOp) ? 1 : c->op_bytes,
2018 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2019 (ctxt->eflags & EFLG_DF),
2020 register_address(c, es_base(ctxt),
2021 c->regs[VCPU_REGS_RDI]),
2023 c->regs[VCPU_REGS_RDX]) == 0) {
2028 case 0x6e: /* outsb */
2029 case 0x6f: /* outsw/outsd */
2030 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2031 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2032 kvm_inject_gp(ctxt->vcpu, 0);
2035 if (kvm_emulate_pio_string(ctxt->vcpu,
2037 (c->d & ByteOp) ? 1 : c->op_bytes,
2039 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2040 (ctxt->eflags & EFLG_DF),
2042 seg_override_base(ctxt, c),
2043 c->regs[VCPU_REGS_RSI]),
2045 c->regs[VCPU_REGS_RDX]) == 0) {
2050 case 0x70 ... 0x7f: /* jcc (short) */
2051 if (test_cc(c->b, ctxt->eflags))
2052 jmp_rel(c, c->src.val);
2054 case 0x80 ... 0x83: /* Grp1 */
2055 switch (c->modrm_reg) {
2075 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2077 case 0x86 ... 0x87: /* xchg */
2079 /* Write back the register source. */
2080 switch (c->dst.bytes) {
2082 *(u8 *) c->src.ptr = (u8) c->dst.val;
2085 *(u16 *) c->src.ptr = (u16) c->dst.val;
2088 *c->src.ptr = (u32) c->dst.val;
2089 break; /* 64b reg: zero-extend */
2091 *c->src.ptr = c->dst.val;
2095 * Write back the memory destination with implicit LOCK
2098 c->dst.val = c->src.val;
2101 case 0x88 ... 0x8b: /* mov */
2103 case 0x8c: { /* mov r/m, sreg */
2104 struct kvm_segment segreg;
2106 if (c->modrm_reg <= 5)
2107 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
2109 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
2111 goto cannot_emulate;
2113 c->dst.val = segreg.selector;
2116 case 0x8d: /* lea r16/r32, m */
2117 c->dst.val = c->modrm_ea;
2119 case 0x8e: { /* mov seg, r/m16 */
2124 if (c->modrm_reg == VCPU_SREG_CS ||
2125 c->modrm_reg > VCPU_SREG_GS) {
2126 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
2130 if (c->modrm_reg == VCPU_SREG_SS)
2131 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
2133 rc = kvm_load_segment_descriptor(ctxt->vcpu, sel, c->modrm_reg);
2135 c->dst.type = OP_NONE; /* Disable writeback. */
2138 case 0x8f: /* pop (sole member of Grp1a) */
2139 rc = emulate_grp1a(ctxt, ops);
2140 if (rc != X86EMUL_CONTINUE)
2143 case 0x90: /* nop / xchg r8,rax */
2144 if (!(c->rex_prefix & 1)) { /* nop */
2145 c->dst.type = OP_NONE;
2148 case 0x91 ... 0x97: /* xchg reg,rax */
2149 c->src.type = c->dst.type = OP_REG;
2150 c->src.bytes = c->dst.bytes = c->op_bytes;
2151 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2152 c->src.val = *(c->src.ptr);
2154 case 0x9c: /* pushf */
2155 c->src.val = (unsigned long) ctxt->eflags;
2158 case 0x9d: /* popf */
2159 c->dst.type = OP_REG;
2160 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2161 c->dst.bytes = c->op_bytes;
2162 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2163 if (rc != X86EMUL_CONTINUE)
2166 case 0xa0 ... 0xa1: /* mov */
2167 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2168 c->dst.val = c->src.val;
2170 case 0xa2 ... 0xa3: /* mov */
2171 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2173 case 0xa4 ... 0xa5: /* movs */
2174 c->dst.type = OP_MEM;
2175 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2176 c->dst.ptr = (unsigned long *)register_address(c,
2178 c->regs[VCPU_REGS_RDI]);
2179 rc = ops->read_emulated(register_address(c,
2180 seg_override_base(ctxt, c),
2181 c->regs[VCPU_REGS_RSI]),
2183 c->dst.bytes, ctxt->vcpu);
2184 if (rc != X86EMUL_CONTINUE)
2186 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2187 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2189 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2190 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2193 case 0xa6 ... 0xa7: /* cmps */
2194 c->src.type = OP_NONE; /* Disable writeback. */
2195 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2196 c->src.ptr = (unsigned long *)register_address(c,
2197 seg_override_base(ctxt, c),
2198 c->regs[VCPU_REGS_RSI]);
2199 rc = ops->read_emulated((unsigned long)c->src.ptr,
2203 if (rc != X86EMUL_CONTINUE)
2206 c->dst.type = OP_NONE; /* Disable writeback. */
2207 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2208 c->dst.ptr = (unsigned long *)register_address(c,
2210 c->regs[VCPU_REGS_RDI]);
2211 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2215 if (rc != X86EMUL_CONTINUE)
2218 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2220 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2222 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2223 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2225 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2226 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2230 case 0xaa ... 0xab: /* stos */
2231 c->dst.type = OP_MEM;
2232 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2233 c->dst.ptr = (unsigned long *)register_address(c,
2235 c->regs[VCPU_REGS_RDI]);
2236 c->dst.val = c->regs[VCPU_REGS_RAX];
2237 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2238 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2241 case 0xac ... 0xad: /* lods */
2242 c->dst.type = OP_REG;
2243 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2244 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2245 rc = ops->read_emulated(register_address(c,
2246 seg_override_base(ctxt, c),
2247 c->regs[VCPU_REGS_RSI]),
2251 if (rc != X86EMUL_CONTINUE)
2253 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2254 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2257 case 0xae ... 0xaf: /* scas */
2258 DPRINTF("Urk! I don't handle SCAS.\n");
2259 goto cannot_emulate;
2260 case 0xb0 ... 0xbf: /* mov r, imm */
2265 case 0xc3: /* ret */
2266 c->dst.type = OP_REG;
2267 c->dst.ptr = &c->eip;
2268 c->dst.bytes = c->op_bytes;
2269 goto pop_instruction;
2270 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2272 c->dst.val = c->src.val;
2274 case 0xcb: /* ret far */
2275 rc = emulate_ret_far(ctxt, ops);
2276 if (rc != X86EMUL_CONTINUE)
2279 case 0xd0 ... 0xd1: /* Grp2 */
2283 case 0xd2 ... 0xd3: /* Grp2 */
2284 c->src.val = c->regs[VCPU_REGS_RCX];
2287 case 0xe4: /* inb */
2292 case 0xe6: /* outb */
2293 case 0xe7: /* out */
2297 case 0xe8: /* call (near) */ {
2298 long int rel = c->src.val;
2299 c->src.val = (unsigned long) c->eip;
2304 case 0xe9: /* jmp rel */
2306 case 0xea: /* jmp far */
2307 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val,
2311 c->eip = c->src.val;
2314 jmp: /* jmp rel short */
2315 jmp_rel(c, c->src.val);
2316 c->dst.type = OP_NONE; /* Disable writeback. */
2318 case 0xec: /* in al,dx */
2319 case 0xed: /* in (e/r)ax,dx */
2320 port = c->regs[VCPU_REGS_RDX];
2323 case 0xee: /* out al,dx */
2324 case 0xef: /* out (e/r)ax,dx */
2325 port = c->regs[VCPU_REGS_RDX];
2328 if (!emulator_io_permited(ctxt, ops, port,
2329 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2330 kvm_inject_gp(ctxt->vcpu, 0);
2333 if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2334 (c->d & ByteOp) ? 1 : c->op_bytes,
2337 goto cannot_emulate;
2340 case 0xf4: /* hlt */
2341 ctxt->vcpu->arch.halt_request = 1;
2343 case 0xf5: /* cmc */
2344 /* complement carry flag from eflags reg */
2345 ctxt->eflags ^= EFLG_CF;
2346 c->dst.type = OP_NONE; /* Disable writeback. */
2348 case 0xf6 ... 0xf7: /* Grp3 */
2349 rc = emulate_grp3(ctxt, ops);
2350 if (rc != X86EMUL_CONTINUE)
2353 case 0xf8: /* clc */
2354 ctxt->eflags &= ~EFLG_CF;
2355 c->dst.type = OP_NONE; /* Disable writeback. */
2357 case 0xfa: /* cli */
2358 if (emulator_bad_iopl(ctxt))
2359 kvm_inject_gp(ctxt->vcpu, 0);
2361 ctxt->eflags &= ~X86_EFLAGS_IF;
2362 c->dst.type = OP_NONE; /* Disable writeback. */
2365 case 0xfb: /* sti */
2366 if (emulator_bad_iopl(ctxt))
2367 kvm_inject_gp(ctxt->vcpu, 0);
2369 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2370 ctxt->eflags |= X86_EFLAGS_IF;
2371 c->dst.type = OP_NONE; /* Disable writeback. */
2374 case 0xfc: /* cld */
2375 ctxt->eflags &= ~EFLG_DF;
2376 c->dst.type = OP_NONE; /* Disable writeback. */
2378 case 0xfd: /* std */
2379 ctxt->eflags |= EFLG_DF;
2380 c->dst.type = OP_NONE; /* Disable writeback. */
2382 case 0xfe ... 0xff: /* Grp4/Grp5 */
2383 rc = emulate_grp45(ctxt, ops);
2384 if (rc != X86EMUL_CONTINUE)
2390 rc = writeback(ctxt, ops);
2391 if (rc != X86EMUL_CONTINUE)
2394 /* Commit shadow register state. */
2395 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2396 kvm_rip_write(ctxt->vcpu, c->eip);
2399 if (rc == X86EMUL_UNHANDLEABLE) {
2407 case 0x01: /* lgdt, lidt, lmsw */
2408 switch (c->modrm_reg) {
2410 unsigned long address;
2412 case 0: /* vmcall */
2413 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2414 goto cannot_emulate;
2416 rc = kvm_fix_hypercall(ctxt->vcpu);
2417 if (rc != X86EMUL_CONTINUE)
2420 /* Let the processor re-execute the fixed hypercall */
2421 c->eip = kvm_rip_read(ctxt->vcpu);
2422 /* Disable writeback. */
2423 c->dst.type = OP_NONE;
2426 rc = read_descriptor(ctxt, ops, c->src.ptr,
2427 &size, &address, c->op_bytes);
2428 if (rc != X86EMUL_CONTINUE)
2430 realmode_lgdt(ctxt->vcpu, size, address);
2431 /* Disable writeback. */
2432 c->dst.type = OP_NONE;
2434 case 3: /* lidt/vmmcall */
2435 if (c->modrm_mod == 3) {
2436 switch (c->modrm_rm) {
2438 rc = kvm_fix_hypercall(ctxt->vcpu);
2439 if (rc != X86EMUL_CONTINUE)
2443 goto cannot_emulate;
2446 rc = read_descriptor(ctxt, ops, c->src.ptr,
2449 if (rc != X86EMUL_CONTINUE)
2451 realmode_lidt(ctxt->vcpu, size, address);
2453 /* Disable writeback. */
2454 c->dst.type = OP_NONE;
2458 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2461 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2463 c->dst.type = OP_NONE;
2466 emulate_invlpg(ctxt->vcpu, memop);
2467 /* Disable writeback. */
2468 c->dst.type = OP_NONE;
2471 goto cannot_emulate;
2474 case 0x05: /* syscall */
2475 rc = emulate_syscall(ctxt);
2476 if (rc != X86EMUL_CONTINUE)
2482 emulate_clts(ctxt->vcpu);
2483 c->dst.type = OP_NONE;
2485 case 0x08: /* invd */
2486 case 0x09: /* wbinvd */
2487 case 0x0d: /* GrpP (prefetch) */
2488 case 0x18: /* Grp16 (prefetch/nop) */
2489 c->dst.type = OP_NONE;
2491 case 0x20: /* mov cr, reg */
2492 if (c->modrm_mod != 3)
2493 goto cannot_emulate;
2494 c->regs[c->modrm_rm] =
2495 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2496 c->dst.type = OP_NONE; /* no writeback */
2498 case 0x21: /* mov from dr to reg */
2499 if (c->modrm_mod != 3)
2500 goto cannot_emulate;
2501 if (emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]))
2502 goto cannot_emulate;
2503 rc = X86EMUL_CONTINUE;
2504 c->dst.type = OP_NONE; /* no writeback */
2506 case 0x22: /* mov reg, cr */
2507 if (c->modrm_mod != 3)
2508 goto cannot_emulate;
2509 realmode_set_cr(ctxt->vcpu,
2510 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2511 c->dst.type = OP_NONE;
2513 case 0x23: /* mov from reg to dr */
2514 if (c->modrm_mod != 3)
2515 goto cannot_emulate;
2516 if (emulator_set_dr(ctxt, c->modrm_reg, c->regs[c->modrm_rm]))
2517 goto cannot_emulate;
2518 rc = X86EMUL_CONTINUE;
2519 c->dst.type = OP_NONE; /* no writeback */
2523 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2524 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2525 if (kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
2526 kvm_inject_gp(ctxt->vcpu, 0);
2527 c->eip = kvm_rip_read(ctxt->vcpu);
2529 rc = X86EMUL_CONTINUE;
2530 c->dst.type = OP_NONE;
2534 if (kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
2535 kvm_inject_gp(ctxt->vcpu, 0);
2536 c->eip = kvm_rip_read(ctxt->vcpu);
2538 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2539 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2541 rc = X86EMUL_CONTINUE;
2542 c->dst.type = OP_NONE;
2544 case 0x34: /* sysenter */
2545 rc = emulate_sysenter(ctxt);
2546 if (rc != X86EMUL_CONTINUE)
2551 case 0x35: /* sysexit */
2552 rc = emulate_sysexit(ctxt);
2553 if (rc != X86EMUL_CONTINUE)
2558 case 0x40 ... 0x4f: /* cmov */
2559 c->dst.val = c->dst.orig_val = c->src.val;
2560 if (!test_cc(c->b, ctxt->eflags))
2561 c->dst.type = OP_NONE; /* no writeback */
2563 case 0x80 ... 0x8f: /* jnz rel, etc*/
2564 if (test_cc(c->b, ctxt->eflags))
2565 jmp_rel(c, c->src.val);
2566 c->dst.type = OP_NONE;
2568 case 0xa0: /* push fs */
2569 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2571 case 0xa1: /* pop fs */
2572 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2573 if (rc != X86EMUL_CONTINUE)
2578 c->dst.type = OP_NONE;
2579 /* only subword offset */
2580 c->src.val &= (c->dst.bytes << 3) - 1;
2581 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2583 case 0xa4: /* shld imm8, r, r/m */
2584 case 0xa5: /* shld cl, r, r/m */
2585 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2587 case 0xa8: /* push gs */
2588 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2590 case 0xa9: /* pop gs */
2591 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2592 if (rc != X86EMUL_CONTINUE)
2597 /* only subword offset */
2598 c->src.val &= (c->dst.bytes << 3) - 1;
2599 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2601 case 0xac: /* shrd imm8, r, r/m */
2602 case 0xad: /* shrd cl, r, r/m */
2603 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2605 case 0xae: /* clflush */
2607 case 0xb0 ... 0xb1: /* cmpxchg */
2609 * Save real source value, then compare EAX against
2612 c->src.orig_val = c->src.val;
2613 c->src.val = c->regs[VCPU_REGS_RAX];
2614 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2615 if (ctxt->eflags & EFLG_ZF) {
2616 /* Success: write back to memory. */
2617 c->dst.val = c->src.orig_val;
2619 /* Failure: write the value we saw to EAX. */
2620 c->dst.type = OP_REG;
2621 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2626 /* only subword offset */
2627 c->src.val &= (c->dst.bytes << 3) - 1;
2628 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2630 case 0xb6 ... 0xb7: /* movzx */
2631 c->dst.bytes = c->op_bytes;
2632 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2635 case 0xba: /* Grp8 */
2636 switch (c->modrm_reg & 3) {
2649 /* only subword offset */
2650 c->src.val &= (c->dst.bytes << 3) - 1;
2651 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2653 case 0xbe ... 0xbf: /* movsx */
2654 c->dst.bytes = c->op_bytes;
2655 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2658 case 0xc3: /* movnti */
2659 c->dst.bytes = c->op_bytes;
2660 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2663 case 0xc7: /* Grp9 (cmpxchg8b) */
2664 rc = emulate_grp9(ctxt, ops, memop);
2665 if (rc != X86EMUL_CONTINUE)
2667 c->dst.type = OP_NONE;
2673 DPRINTF("Cannot emulate %02x\n", c->b);