1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
79 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
80 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
82 /* Source 2 operand type */
83 #define Src2None (0<<29)
84 #define Src2CL (1<<29)
85 #define Src2ImmByte (2<<29)
86 #define Src2One (3<<29)
87 #define Src2Imm16 (4<<29)
88 #define Src2Mask (7<<29)
91 Group1_80, Group1_81, Group1_82, Group1_83,
92 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
96 static u32 opcode_table[256] = {
98 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
99 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
100 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
101 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
103 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
104 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
105 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
106 ImplicitOps | Stack | No64, 0,
108 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
109 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
110 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
111 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
113 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
114 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
115 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
116 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
118 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
119 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
120 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
122 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
123 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
126 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
127 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
131 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
132 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
135 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
137 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
139 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
140 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
142 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
143 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
145 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
146 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
149 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
150 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
151 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
153 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
156 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
159 Group | Group1_80, Group | Group1_81,
160 Group | Group1_82, Group | Group1_83,
161 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
162 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
164 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
165 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
166 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
167 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
169 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
171 0, 0, SrcImm | Src2Imm16 | No64, 0,
172 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
174 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
175 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
176 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
177 ByteOp | ImplicitOps | String, ImplicitOps | String,
179 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
180 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
181 ByteOp | ImplicitOps | String, ImplicitOps | String,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
184 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
185 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
186 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
189 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
190 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
191 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
193 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
194 0, ImplicitOps | Stack, 0, 0,
195 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
197 0, 0, 0, ImplicitOps | Stack,
198 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
200 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
201 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
204 0, 0, 0, 0, 0, 0, 0, 0,
207 ByteOp | SrcImmUByte, SrcImmUByte,
208 ByteOp | SrcImmUByte, SrcImmUByte,
210 SrcImm | Stack, SrcImm | ImplicitOps,
211 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
212 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
213 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
216 ImplicitOps | Priv, ImplicitOps, Group | Group3_Byte, Group | Group3,
218 ImplicitOps, 0, ImplicitOps, ImplicitOps,
219 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
222 static u32 twobyte_table[256] = {
224 0, Group | GroupDual | Group7, 0, 0,
225 0, ImplicitOps, ImplicitOps | Priv, 0,
226 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
227 0, ImplicitOps | ModRM, 0, 0,
229 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
231 ModRM | ImplicitOps | Priv, ModRM | Priv,
232 ModRM | ImplicitOps | Priv, ModRM | Priv,
234 0, 0, 0, 0, 0, 0, 0, 0,
236 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
237 ImplicitOps, ImplicitOps | Priv, 0, 0,
238 0, 0, 0, 0, 0, 0, 0, 0,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
243 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
245 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
246 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
247 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
248 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
250 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
252 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
254 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
256 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
257 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
259 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
261 ImplicitOps | Stack, ImplicitOps | Stack,
262 0, DstMem | SrcReg | ModRM | BitOp,
263 DstMem | SrcReg | Src2ImmByte | ModRM,
264 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
266 ImplicitOps | Stack, ImplicitOps | Stack,
267 0, DstMem | SrcReg | ModRM | BitOp | Lock,
268 DstMem | SrcReg | Src2ImmByte | ModRM,
269 DstMem | SrcReg | Src2CL | ModRM,
272 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
273 0, DstMem | SrcReg | ModRM | BitOp | Lock,
274 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
275 DstReg | SrcMem16 | ModRM | Mov,
278 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
279 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
280 DstReg | SrcMem16 | ModRM | Mov,
282 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
283 0, 0, 0, Group | GroupDual | Group9,
284 0, 0, 0, 0, 0, 0, 0, 0,
286 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
288 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
290 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
293 static u32 group_table[] = {
295 ByteOp | DstMem | SrcImm | ModRM | Lock,
296 ByteOp | DstMem | SrcImm | ModRM | Lock,
297 ByteOp | DstMem | SrcImm | ModRM | Lock,
298 ByteOp | DstMem | SrcImm | ModRM | Lock,
299 ByteOp | DstMem | SrcImm | ModRM | Lock,
300 ByteOp | DstMem | SrcImm | ModRM | Lock,
301 ByteOp | DstMem | SrcImm | ModRM | Lock,
302 ByteOp | DstMem | SrcImm | ModRM,
304 DstMem | SrcImm | ModRM | Lock,
305 DstMem | SrcImm | ModRM | Lock,
306 DstMem | SrcImm | ModRM | Lock,
307 DstMem | SrcImm | ModRM | Lock,
308 DstMem | SrcImm | ModRM | Lock,
309 DstMem | SrcImm | ModRM | Lock,
310 DstMem | SrcImm | ModRM | Lock,
311 DstMem | SrcImm | ModRM,
313 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
314 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
315 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
316 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
317 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
318 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
319 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
320 ByteOp | DstMem | SrcImm | ModRM | No64,
322 DstMem | SrcImmByte | ModRM | Lock,
323 DstMem | SrcImmByte | ModRM | Lock,
324 DstMem | SrcImmByte | ModRM | Lock,
325 DstMem | SrcImmByte | ModRM | Lock,
326 DstMem | SrcImmByte | ModRM | Lock,
327 DstMem | SrcImmByte | ModRM | Lock,
328 DstMem | SrcImmByte | ModRM | Lock,
329 DstMem | SrcImmByte | ModRM,
331 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
333 ByteOp | SrcImm | DstMem | ModRM, 0,
334 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
337 DstMem | SrcImm | ModRM, 0,
338 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
341 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
344 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
345 SrcMem | ModRM | Stack, 0,
346 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
348 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
349 SrcNone | ModRM | DstMem | Mov, 0,
350 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
353 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
354 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
356 0, ImplicitOps | ModRM | Lock, 0, 0, 0, 0, 0, 0,
359 static u32 group2_table[] = {
361 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM,
362 SrcNone | ModRM | DstMem | Mov, 0,
363 SrcMem16 | ModRM | Mov, 0,
365 0, 0, 0, 0, 0, 0, 0, 0,
368 /* EFLAGS bit definitions. */
369 #define EFLG_ID (1<<21)
370 #define EFLG_VIP (1<<20)
371 #define EFLG_VIF (1<<19)
372 #define EFLG_AC (1<<18)
373 #define EFLG_VM (1<<17)
374 #define EFLG_RF (1<<16)
375 #define EFLG_IOPL (3<<12)
376 #define EFLG_NT (1<<14)
377 #define EFLG_OF (1<<11)
378 #define EFLG_DF (1<<10)
379 #define EFLG_IF (1<<9)
380 #define EFLG_TF (1<<8)
381 #define EFLG_SF (1<<7)
382 #define EFLG_ZF (1<<6)
383 #define EFLG_AF (1<<4)
384 #define EFLG_PF (1<<2)
385 #define EFLG_CF (1<<0)
388 * Instruction emulation:
389 * Most instructions are emulated directly via a fragment of inline assembly
390 * code. This allows us to save/restore EFLAGS and thus very easily pick up
391 * any modified flags.
394 #if defined(CONFIG_X86_64)
395 #define _LO32 "k" /* force 32-bit operand */
396 #define _STK "%%rsp" /* stack pointer */
397 #elif defined(__i386__)
398 #define _LO32 "" /* force 32-bit operand */
399 #define _STK "%%esp" /* stack pointer */
403 * These EFLAGS bits are restored from saved value during emulation, and
404 * any changes are written back to the saved value after emulation.
406 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
408 /* Before executing instruction: restore necessary bits in EFLAGS. */
409 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
410 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
411 "movl %"_sav",%"_LO32 _tmp"; " \
414 "movl %"_msk",%"_LO32 _tmp"; " \
415 "andl %"_LO32 _tmp",("_STK"); " \
417 "notl %"_LO32 _tmp"; " \
418 "andl %"_LO32 _tmp",("_STK"); " \
419 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
421 "orl %"_LO32 _tmp",("_STK"); " \
425 /* After executing instruction: write-back necessary bits in EFLAGS. */
426 #define _POST_EFLAGS(_sav, _msk, _tmp) \
427 /* _sav |= EFLAGS & _msk; */ \
430 "andl %"_msk",%"_LO32 _tmp"; " \
431 "orl %"_LO32 _tmp",%"_sav"; "
439 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
441 __asm__ __volatile__ ( \
442 _PRE_EFLAGS("0", "4", "2") \
443 _op _suffix " %"_x"3,%1; " \
444 _POST_EFLAGS("0", "4", "2") \
445 : "=m" (_eflags), "=m" ((_dst).val), \
447 : _y ((_src).val), "i" (EFLAGS_MASK)); \
451 /* Raw emulation: instruction has two explicit operands. */
452 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
454 unsigned long _tmp; \
456 switch ((_dst).bytes) { \
458 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
461 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
464 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
469 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
471 unsigned long _tmp; \
472 switch ((_dst).bytes) { \
474 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
477 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
478 _wx, _wy, _lx, _ly, _qx, _qy); \
483 /* Source operand is byte-sized and may be restricted to just %cl. */
484 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
485 __emulate_2op(_op, _src, _dst, _eflags, \
486 "b", "c", "b", "c", "b", "c", "b", "c")
488 /* Source operand is byte, word, long or quad sized. */
489 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
490 __emulate_2op(_op, _src, _dst, _eflags, \
491 "b", "q", "w", "r", _LO32, "r", "", "r")
493 /* Source operand is word, long or quad sized. */
494 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
495 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
496 "w", "r", _LO32, "r", "", "r")
498 /* Instruction has three operands and one operand is stored in ECX register */
499 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
501 unsigned long _tmp; \
502 _type _clv = (_cl).val; \
503 _type _srcv = (_src).val; \
504 _type _dstv = (_dst).val; \
506 __asm__ __volatile__ ( \
507 _PRE_EFLAGS("0", "5", "2") \
508 _op _suffix " %4,%1 \n" \
509 _POST_EFLAGS("0", "5", "2") \
510 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
511 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
514 (_cl).val = (unsigned long) _clv; \
515 (_src).val = (unsigned long) _srcv; \
516 (_dst).val = (unsigned long) _dstv; \
519 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
521 switch ((_dst).bytes) { \
523 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
524 "w", unsigned short); \
527 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
528 "l", unsigned int); \
531 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
532 "q", unsigned long)); \
537 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
539 unsigned long _tmp; \
541 __asm__ __volatile__ ( \
542 _PRE_EFLAGS("0", "3", "2") \
543 _op _suffix " %1; " \
544 _POST_EFLAGS("0", "3", "2") \
545 : "=m" (_eflags), "+m" ((_dst).val), \
547 : "i" (EFLAGS_MASK)); \
550 /* Instruction has only one explicit operand (no source operand). */
551 #define emulate_1op(_op, _dst, _eflags) \
553 switch ((_dst).bytes) { \
554 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
555 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
556 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
557 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
561 /* Fetch next part of the instruction being emulated. */
562 #define insn_fetch(_type, _size, _eip) \
563 ({ unsigned long _x; \
564 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
571 static inline unsigned long ad_mask(struct decode_cache *c)
573 return (1UL << (c->ad_bytes << 3)) - 1;
576 /* Access/update address held in a register, based on addressing mode. */
577 static inline unsigned long
578 address_mask(struct decode_cache *c, unsigned long reg)
580 if (c->ad_bytes == sizeof(unsigned long))
583 return reg & ad_mask(c);
586 static inline unsigned long
587 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
589 return base + address_mask(c, reg);
593 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
595 if (c->ad_bytes == sizeof(unsigned long))
598 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
601 static inline void jmp_rel(struct decode_cache *c, int rel)
603 register_address_increment(c, &c->eip, rel);
606 static void set_seg_override(struct decode_cache *c, int seg)
608 c->has_seg_override = true;
609 c->seg_override = seg;
612 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
614 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
617 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
620 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
621 struct decode_cache *c)
623 if (!c->has_seg_override)
626 return seg_base(ctxt, c->seg_override);
629 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
631 return seg_base(ctxt, VCPU_SREG_ES);
634 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
636 return seg_base(ctxt, VCPU_SREG_SS);
639 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
640 struct x86_emulate_ops *ops,
641 unsigned long linear, u8 *dest)
643 struct fetch_cache *fc = &ctxt->decode.fetch;
647 if (linear < fc->start || linear >= fc->end) {
648 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
649 rc = ops->fetch(linear, fc->data, size, ctxt->vcpu, NULL);
653 fc->end = linear + size;
655 *dest = fc->data[linear - fc->start];
659 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
660 struct x86_emulate_ops *ops,
661 unsigned long eip, void *dest, unsigned size)
665 /* x86 instructions are limited to 15 bytes. */
666 if (eip + size - ctxt->decode.eip_orig > 15)
667 return X86EMUL_UNHANDLEABLE;
668 eip += ctxt->cs_base;
670 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
678 * Given the 'reg' portion of a ModRM byte, and a register block, return a
679 * pointer into the block that addresses the relevant register.
680 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
682 static void *decode_register(u8 modrm_reg, unsigned long *regs,
687 p = ®s[modrm_reg];
688 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
689 p = (unsigned char *)®s[modrm_reg & 3] + 1;
693 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
694 struct x86_emulate_ops *ops,
696 u16 *size, unsigned long *address, int op_bytes)
703 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
707 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
712 static int test_cc(unsigned int condition, unsigned int flags)
716 switch ((condition & 15) >> 1) {
718 rc |= (flags & EFLG_OF);
720 case 1: /* b/c/nae */
721 rc |= (flags & EFLG_CF);
724 rc |= (flags & EFLG_ZF);
727 rc |= (flags & (EFLG_CF|EFLG_ZF));
730 rc |= (flags & EFLG_SF);
733 rc |= (flags & EFLG_PF);
736 rc |= (flags & EFLG_ZF);
739 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
743 /* Odd condition identifiers (lsb == 1) have inverted sense. */
744 return (!!rc ^ (condition & 1));
747 static void decode_register_operand(struct operand *op,
748 struct decode_cache *c,
751 unsigned reg = c->modrm_reg;
752 int highbyte_regs = c->rex_prefix == 0;
755 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
757 if ((c->d & ByteOp) && !inhibit_bytereg) {
758 op->ptr = decode_register(reg, c->regs, highbyte_regs);
759 op->val = *(u8 *)op->ptr;
762 op->ptr = decode_register(reg, c->regs, 0);
763 op->bytes = c->op_bytes;
766 op->val = *(u16 *)op->ptr;
769 op->val = *(u32 *)op->ptr;
772 op->val = *(u64 *) op->ptr;
776 op->orig_val = op->val;
779 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
780 struct x86_emulate_ops *ops)
782 struct decode_cache *c = &ctxt->decode;
784 int index_reg = 0, base_reg = 0, scale;
788 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
789 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
790 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
793 c->modrm = insn_fetch(u8, 1, c->eip);
794 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
795 c->modrm_reg |= (c->modrm & 0x38) >> 3;
796 c->modrm_rm |= (c->modrm & 0x07);
800 if (c->modrm_mod == 3) {
801 c->modrm_ptr = decode_register(c->modrm_rm,
802 c->regs, c->d & ByteOp);
803 c->modrm_val = *(unsigned long *)c->modrm_ptr;
807 if (c->ad_bytes == 2) {
808 unsigned bx = c->regs[VCPU_REGS_RBX];
809 unsigned bp = c->regs[VCPU_REGS_RBP];
810 unsigned si = c->regs[VCPU_REGS_RSI];
811 unsigned di = c->regs[VCPU_REGS_RDI];
813 /* 16-bit ModR/M decode. */
814 switch (c->modrm_mod) {
816 if (c->modrm_rm == 6)
817 c->modrm_ea += insn_fetch(u16, 2, c->eip);
820 c->modrm_ea += insn_fetch(s8, 1, c->eip);
823 c->modrm_ea += insn_fetch(u16, 2, c->eip);
826 switch (c->modrm_rm) {
828 c->modrm_ea += bx + si;
831 c->modrm_ea += bx + di;
834 c->modrm_ea += bp + si;
837 c->modrm_ea += bp + di;
846 if (c->modrm_mod != 0)
853 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
854 (c->modrm_rm == 6 && c->modrm_mod != 0))
855 if (!c->has_seg_override)
856 set_seg_override(c, VCPU_SREG_SS);
857 c->modrm_ea = (u16)c->modrm_ea;
859 /* 32/64-bit ModR/M decode. */
860 if ((c->modrm_rm & 7) == 4) {
861 sib = insn_fetch(u8, 1, c->eip);
862 index_reg |= (sib >> 3) & 7;
866 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
867 c->modrm_ea += insn_fetch(s32, 4, c->eip);
869 c->modrm_ea += c->regs[base_reg];
871 c->modrm_ea += c->regs[index_reg] << scale;
872 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
873 if (ctxt->mode == X86EMUL_MODE_PROT64)
876 c->modrm_ea += c->regs[c->modrm_rm];
877 switch (c->modrm_mod) {
879 if (c->modrm_rm == 5)
880 c->modrm_ea += insn_fetch(s32, 4, c->eip);
883 c->modrm_ea += insn_fetch(s8, 1, c->eip);
886 c->modrm_ea += insn_fetch(s32, 4, c->eip);
894 static int decode_abs(struct x86_emulate_ctxt *ctxt,
895 struct x86_emulate_ops *ops)
897 struct decode_cache *c = &ctxt->decode;
900 switch (c->ad_bytes) {
902 c->modrm_ea = insn_fetch(u16, 2, c->eip);
905 c->modrm_ea = insn_fetch(u32, 4, c->eip);
908 c->modrm_ea = insn_fetch(u64, 8, c->eip);
916 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
918 struct decode_cache *c = &ctxt->decode;
920 int mode = ctxt->mode;
921 int def_op_bytes, def_ad_bytes, group;
923 /* Shadow copy of register state. Committed on successful emulation. */
925 memset(c, 0, sizeof(struct decode_cache));
926 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
927 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
928 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
931 case X86EMUL_MODE_REAL:
932 case X86EMUL_MODE_VM86:
933 case X86EMUL_MODE_PROT16:
934 def_op_bytes = def_ad_bytes = 2;
936 case X86EMUL_MODE_PROT32:
937 def_op_bytes = def_ad_bytes = 4;
940 case X86EMUL_MODE_PROT64:
949 c->op_bytes = def_op_bytes;
950 c->ad_bytes = def_ad_bytes;
952 /* Legacy prefixes. */
954 switch (c->b = insn_fetch(u8, 1, c->eip)) {
955 case 0x66: /* operand-size override */
956 /* switch between 2/4 bytes */
957 c->op_bytes = def_op_bytes ^ 6;
959 case 0x67: /* address-size override */
960 if (mode == X86EMUL_MODE_PROT64)
961 /* switch between 4/8 bytes */
962 c->ad_bytes = def_ad_bytes ^ 12;
964 /* switch between 2/4 bytes */
965 c->ad_bytes = def_ad_bytes ^ 6;
967 case 0x26: /* ES override */
968 case 0x2e: /* CS override */
969 case 0x36: /* SS override */
970 case 0x3e: /* DS override */
971 set_seg_override(c, (c->b >> 3) & 3);
973 case 0x64: /* FS override */
974 case 0x65: /* GS override */
975 set_seg_override(c, c->b & 7);
977 case 0x40 ... 0x4f: /* REX */
978 if (mode != X86EMUL_MODE_PROT64)
980 c->rex_prefix = c->b;
982 case 0xf0: /* LOCK */
985 case 0xf2: /* REPNE/REPNZ */
986 c->rep_prefix = REPNE_PREFIX;
988 case 0xf3: /* REP/REPE/REPZ */
989 c->rep_prefix = REPE_PREFIX;
995 /* Any legacy prefix after a REX prefix nullifies its effect. */
1004 if (c->rex_prefix & 8)
1005 c->op_bytes = 8; /* REX.W */
1007 /* Opcode byte(s). */
1008 c->d = opcode_table[c->b];
1010 /* Two-byte opcode? */
1013 c->b = insn_fetch(u8, 1, c->eip);
1014 c->d = twobyte_table[c->b];
1019 group = c->d & GroupMask;
1020 c->modrm = insn_fetch(u8, 1, c->eip);
1023 group = (group << 3) + ((c->modrm >> 3) & 7);
1024 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1025 c->d = group2_table[group];
1027 c->d = group_table[group];
1032 DPRINTF("Cannot emulate %02x\n", c->b);
1036 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1039 /* ModRM and SIB bytes. */
1041 rc = decode_modrm(ctxt, ops);
1042 else if (c->d & MemAbs)
1043 rc = decode_abs(ctxt, ops);
1047 if (!c->has_seg_override)
1048 set_seg_override(c, VCPU_SREG_DS);
1050 if (!(!c->twobyte && c->b == 0x8d))
1051 c->modrm_ea += seg_override_base(ctxt, c);
1053 if (c->ad_bytes != 8)
1054 c->modrm_ea = (u32)c->modrm_ea;
1056 * Decode and fetch the source operand: register, memory
1059 switch (c->d & SrcMask) {
1063 decode_register_operand(&c->src, c, 0);
1072 c->src.bytes = (c->d & ByteOp) ? 1 :
1074 /* Don't fetch the address for invlpg: it could be unmapped. */
1075 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1079 * For instructions with a ModR/M byte, switch to register
1080 * access if Mod = 3.
1082 if ((c->d & ModRM) && c->modrm_mod == 3) {
1083 c->src.type = OP_REG;
1084 c->src.val = c->modrm_val;
1085 c->src.ptr = c->modrm_ptr;
1088 c->src.type = OP_MEM;
1092 c->src.type = OP_IMM;
1093 c->src.ptr = (unsigned long *)c->eip;
1094 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1095 if (c->src.bytes == 8)
1097 /* NB. Immediates are sign-extended as necessary. */
1098 switch (c->src.bytes) {
1100 c->src.val = insn_fetch(s8, 1, c->eip);
1103 c->src.val = insn_fetch(s16, 2, c->eip);
1106 c->src.val = insn_fetch(s32, 4, c->eip);
1109 if ((c->d & SrcMask) == SrcImmU) {
1110 switch (c->src.bytes) {
1115 c->src.val &= 0xffff;
1118 c->src.val &= 0xffffffff;
1125 c->src.type = OP_IMM;
1126 c->src.ptr = (unsigned long *)c->eip;
1128 if ((c->d & SrcMask) == SrcImmByte)
1129 c->src.val = insn_fetch(s8, 1, c->eip);
1131 c->src.val = insn_fetch(u8, 1, c->eip);
1140 * Decode and fetch the second source operand: register, memory
1143 switch (c->d & Src2Mask) {
1148 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1151 c->src2.type = OP_IMM;
1152 c->src2.ptr = (unsigned long *)c->eip;
1154 c->src2.val = insn_fetch(u8, 1, c->eip);
1157 c->src2.type = OP_IMM;
1158 c->src2.ptr = (unsigned long *)c->eip;
1160 c->src2.val = insn_fetch(u16, 2, c->eip);
1168 /* Decode and fetch the destination operand: register or memory. */
1169 switch (c->d & DstMask) {
1171 /* Special instructions do their own operand decoding. */
1174 decode_register_operand(&c->dst, c,
1175 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1178 if ((c->d & ModRM) && c->modrm_mod == 3) {
1179 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1180 c->dst.type = OP_REG;
1181 c->dst.val = c->dst.orig_val = c->modrm_val;
1182 c->dst.ptr = c->modrm_ptr;
1185 c->dst.type = OP_MEM;
1188 c->dst.type = OP_REG;
1189 c->dst.bytes = c->op_bytes;
1190 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1191 switch (c->op_bytes) {
1193 c->dst.val = *(u8 *)c->dst.ptr;
1196 c->dst.val = *(u16 *)c->dst.ptr;
1199 c->dst.val = *(u32 *)c->dst.ptr;
1202 c->dst.orig_val = c->dst.val;
1206 if (c->rip_relative)
1207 c->modrm_ea += c->eip;
1210 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1213 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1215 struct decode_cache *c = &ctxt->decode;
1217 c->dst.type = OP_MEM;
1218 c->dst.bytes = c->op_bytes;
1219 c->dst.val = c->src.val;
1220 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1221 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1222 c->regs[VCPU_REGS_RSP]);
1225 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1226 struct x86_emulate_ops *ops,
1227 void *dest, int len)
1229 struct decode_cache *c = &ctxt->decode;
1232 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1233 c->regs[VCPU_REGS_RSP]),
1234 dest, len, ctxt->vcpu);
1235 if (rc != X86EMUL_CONTINUE)
1238 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1242 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1243 struct x86_emulate_ops *ops,
1244 void *dest, int len)
1247 unsigned long val, change_mask;
1248 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1249 int cpl = kvm_x86_ops->get_cpl(ctxt->vcpu);
1251 rc = emulate_pop(ctxt, ops, &val, len);
1252 if (rc != X86EMUL_CONTINUE)
1255 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1256 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1258 switch(ctxt->mode) {
1259 case X86EMUL_MODE_PROT64:
1260 case X86EMUL_MODE_PROT32:
1261 case X86EMUL_MODE_PROT16:
1263 change_mask |= EFLG_IOPL;
1265 change_mask |= EFLG_IF;
1267 case X86EMUL_MODE_VM86:
1269 kvm_inject_gp(ctxt->vcpu, 0);
1270 return X86EMUL_PROPAGATE_FAULT;
1272 change_mask |= EFLG_IF;
1274 default: /* real mode */
1275 change_mask |= (EFLG_IOPL | EFLG_IF);
1279 *(unsigned long *)dest =
1280 (ctxt->eflags & ~change_mask) | (val & change_mask);
1285 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1287 struct decode_cache *c = &ctxt->decode;
1288 struct kvm_segment segment;
1290 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1292 c->src.val = segment.selector;
1296 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1297 struct x86_emulate_ops *ops, int seg)
1299 struct decode_cache *c = &ctxt->decode;
1300 unsigned long selector;
1303 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1307 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, seg);
1311 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1313 struct decode_cache *c = &ctxt->decode;
1314 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1315 int reg = VCPU_REGS_RAX;
1317 while (reg <= VCPU_REGS_RDI) {
1318 (reg == VCPU_REGS_RSP) ?
1319 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1326 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1327 struct x86_emulate_ops *ops)
1329 struct decode_cache *c = &ctxt->decode;
1331 int reg = VCPU_REGS_RDI;
1333 while (reg >= VCPU_REGS_RAX) {
1334 if (reg == VCPU_REGS_RSP) {
1335 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1340 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1348 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1349 struct x86_emulate_ops *ops)
1351 struct decode_cache *c = &ctxt->decode;
1354 rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1360 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1362 struct decode_cache *c = &ctxt->decode;
1363 switch (c->modrm_reg) {
1365 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1368 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1371 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1374 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1376 case 4: /* sal/shl */
1377 case 6: /* sal/shl */
1378 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1381 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1384 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1389 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1390 struct x86_emulate_ops *ops)
1392 struct decode_cache *c = &ctxt->decode;
1395 switch (c->modrm_reg) {
1396 case 0 ... 1: /* test */
1397 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1400 c->dst.val = ~c->dst.val;
1403 emulate_1op("neg", c->dst, ctxt->eflags);
1406 DPRINTF("Cannot emulate %02x\n", c->b);
1407 rc = X86EMUL_UNHANDLEABLE;
1413 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1414 struct x86_emulate_ops *ops)
1416 struct decode_cache *c = &ctxt->decode;
1418 switch (c->modrm_reg) {
1420 emulate_1op("inc", c->dst, ctxt->eflags);
1423 emulate_1op("dec", c->dst, ctxt->eflags);
1425 case 2: /* call near abs */ {
1428 c->eip = c->src.val;
1429 c->src.val = old_eip;
1433 case 4: /* jmp abs */
1434 c->eip = c->src.val;
1443 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1444 struct x86_emulate_ops *ops,
1445 unsigned long memop)
1447 struct decode_cache *c = &ctxt->decode;
1451 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1452 if (rc != X86EMUL_CONTINUE)
1455 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1456 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1458 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1459 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1460 ctxt->eflags &= ~EFLG_ZF;
1463 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1464 (u32) c->regs[VCPU_REGS_RBX];
1466 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1467 if (rc != X86EMUL_CONTINUE)
1469 ctxt->eflags |= EFLG_ZF;
1474 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1475 struct x86_emulate_ops *ops)
1477 struct decode_cache *c = &ctxt->decode;
1481 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1484 if (c->op_bytes == 4)
1485 c->eip = (u32)c->eip;
1486 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1489 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, VCPU_SREG_CS);
1493 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1494 struct x86_emulate_ops *ops)
1497 struct decode_cache *c = &ctxt->decode;
1499 switch (c->dst.type) {
1501 /* The 4-byte case *is* correct:
1502 * in 64-bit mode we zero-extend.
1504 switch (c->dst.bytes) {
1506 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1509 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1512 *c->dst.ptr = (u32)c->dst.val;
1513 break; /* 64b: zero-ext */
1515 *c->dst.ptr = c->dst.val;
1521 rc = ops->cmpxchg_emulated(
1522 (unsigned long)c->dst.ptr,
1528 rc = ops->write_emulated(
1529 (unsigned long)c->dst.ptr,
1533 if (rc != X86EMUL_CONTINUE)
1545 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1547 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1549 * an sti; sti; sequence only disable interrupts for the first
1550 * instruction. So, if the last instruction, be it emulated or
1551 * not, left the system with the INT_STI flag enabled, it
1552 * means that the last instruction is an sti. We should not
1553 * leave the flag on in this case. The same goes for mov ss
1555 if (!(int_shadow & mask))
1556 ctxt->interruptibility = mask;
1560 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1561 struct kvm_segment *cs, struct kvm_segment *ss)
1563 memset(cs, 0, sizeof(struct kvm_segment));
1564 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1565 memset(ss, 0, sizeof(struct kvm_segment));
1567 cs->l = 0; /* will be adjusted later */
1568 cs->base = 0; /* flat segment */
1569 cs->g = 1; /* 4kb granularity */
1570 cs->limit = 0xffffffff; /* 4GB limit */
1571 cs->type = 0x0b; /* Read, Execute, Accessed */
1573 cs->dpl = 0; /* will be adjusted later */
1578 ss->base = 0; /* flat segment */
1579 ss->limit = 0xffffffff; /* 4GB limit */
1580 ss->g = 1; /* 4kb granularity */
1582 ss->type = 0x03; /* Read/Write, Accessed */
1583 ss->db = 1; /* 32bit stack segment */
1589 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1591 struct decode_cache *c = &ctxt->decode;
1592 struct kvm_segment cs, ss;
1595 /* syscall is not available in real mode */
1596 if (ctxt->mode == X86EMUL_MODE_REAL || ctxt->mode == X86EMUL_MODE_VM86)
1597 return X86EMUL_UNHANDLEABLE;
1599 setup_syscalls_segments(ctxt, &cs, &ss);
1601 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1603 cs.selector = (u16)(msr_data & 0xfffc);
1604 ss.selector = (u16)(msr_data + 8);
1606 if (is_long_mode(ctxt->vcpu)) {
1610 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1611 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1613 c->regs[VCPU_REGS_RCX] = c->eip;
1614 if (is_long_mode(ctxt->vcpu)) {
1615 #ifdef CONFIG_X86_64
1616 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1618 kvm_x86_ops->get_msr(ctxt->vcpu,
1619 ctxt->mode == X86EMUL_MODE_PROT64 ?
1620 MSR_LSTAR : MSR_CSTAR, &msr_data);
1623 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1624 ctxt->eflags &= ~(msr_data | EFLG_RF);
1628 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1629 c->eip = (u32)msr_data;
1631 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1634 return X86EMUL_CONTINUE;
1638 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1640 struct decode_cache *c = &ctxt->decode;
1641 struct kvm_segment cs, ss;
1644 /* inject #GP if in real mode */
1645 if (ctxt->mode == X86EMUL_MODE_REAL) {
1646 kvm_inject_gp(ctxt->vcpu, 0);
1647 return X86EMUL_UNHANDLEABLE;
1650 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1651 * Therefore, we inject an #UD.
1653 if (ctxt->mode == X86EMUL_MODE_PROT64)
1654 return X86EMUL_UNHANDLEABLE;
1656 setup_syscalls_segments(ctxt, &cs, &ss);
1658 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1659 switch (ctxt->mode) {
1660 case X86EMUL_MODE_PROT32:
1661 if ((msr_data & 0xfffc) == 0x0) {
1662 kvm_inject_gp(ctxt->vcpu, 0);
1663 return X86EMUL_PROPAGATE_FAULT;
1666 case X86EMUL_MODE_PROT64:
1667 if (msr_data == 0x0) {
1668 kvm_inject_gp(ctxt->vcpu, 0);
1669 return X86EMUL_PROPAGATE_FAULT;
1674 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1675 cs.selector = (u16)msr_data;
1676 cs.selector &= ~SELECTOR_RPL_MASK;
1677 ss.selector = cs.selector + 8;
1678 ss.selector &= ~SELECTOR_RPL_MASK;
1679 if (ctxt->mode == X86EMUL_MODE_PROT64
1680 || is_long_mode(ctxt->vcpu)) {
1685 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1686 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1688 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1691 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1692 c->regs[VCPU_REGS_RSP] = msr_data;
1694 return X86EMUL_CONTINUE;
1698 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1700 struct decode_cache *c = &ctxt->decode;
1701 struct kvm_segment cs, ss;
1705 /* inject #GP if in real mode or Virtual 8086 mode */
1706 if (ctxt->mode == X86EMUL_MODE_REAL ||
1707 ctxt->mode == X86EMUL_MODE_VM86) {
1708 kvm_inject_gp(ctxt->vcpu, 0);
1709 return X86EMUL_UNHANDLEABLE;
1712 setup_syscalls_segments(ctxt, &cs, &ss);
1714 if ((c->rex_prefix & 0x8) != 0x0)
1715 usermode = X86EMUL_MODE_PROT64;
1717 usermode = X86EMUL_MODE_PROT32;
1721 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1723 case X86EMUL_MODE_PROT32:
1724 cs.selector = (u16)(msr_data + 16);
1725 if ((msr_data & 0xfffc) == 0x0) {
1726 kvm_inject_gp(ctxt->vcpu, 0);
1727 return X86EMUL_PROPAGATE_FAULT;
1729 ss.selector = (u16)(msr_data + 24);
1731 case X86EMUL_MODE_PROT64:
1732 cs.selector = (u16)(msr_data + 32);
1733 if (msr_data == 0x0) {
1734 kvm_inject_gp(ctxt->vcpu, 0);
1735 return X86EMUL_PROPAGATE_FAULT;
1737 ss.selector = cs.selector + 8;
1742 cs.selector |= SELECTOR_RPL_MASK;
1743 ss.selector |= SELECTOR_RPL_MASK;
1745 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1746 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1748 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1749 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1751 return X86EMUL_CONTINUE;
1754 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
1757 if (ctxt->mode == X86EMUL_MODE_REAL)
1759 if (ctxt->mode == X86EMUL_MODE_VM86)
1761 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1762 return kvm_x86_ops->get_cpl(ctxt->vcpu) > iopl;
1765 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1766 struct x86_emulate_ops *ops,
1769 struct kvm_segment tr_seg;
1772 u8 perm, bit_idx = port & 0x7;
1773 unsigned mask = (1 << len) - 1;
1775 kvm_get_segment(ctxt->vcpu, &tr_seg, VCPU_SREG_TR);
1776 if (tr_seg.unusable)
1778 if (tr_seg.limit < 103)
1780 r = ops->read_std(tr_seg.base + 102, &io_bitmap_ptr, 2, ctxt->vcpu,
1782 if (r != X86EMUL_CONTINUE)
1784 if (io_bitmap_ptr + port/8 > tr_seg.limit)
1786 r = ops->read_std(tr_seg.base + io_bitmap_ptr + port/8, &perm, 1,
1788 if (r != X86EMUL_CONTINUE)
1790 if ((perm >> bit_idx) & mask)
1795 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1796 struct x86_emulate_ops *ops,
1799 if (emulator_bad_iopl(ctxt))
1800 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1806 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1808 unsigned long memop = 0;
1810 unsigned long saved_eip = 0;
1811 struct decode_cache *c = &ctxt->decode;
1816 ctxt->interruptibility = 0;
1818 /* Shadow copy of register state. Committed on successful emulation.
1819 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1823 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1826 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
1827 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
1831 /* LOCK prefix is allowed only with some instructions */
1832 if (c->lock_prefix && !(c->d & Lock)) {
1833 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
1837 /* Privileged instruction can be executed only in CPL=0 */
1838 if ((c->d & Priv) && kvm_x86_ops->get_cpl(ctxt->vcpu)) {
1839 kvm_inject_gp(ctxt->vcpu, 0);
1843 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1844 memop = c->modrm_ea;
1846 if (c->rep_prefix && (c->d & String)) {
1847 /* All REP prefixes have the same first termination condition */
1848 if (c->regs[VCPU_REGS_RCX] == 0) {
1849 kvm_rip_write(ctxt->vcpu, c->eip);
1852 /* The second termination condition only applies for REPE
1853 * and REPNE. Test if the repeat string operation prefix is
1854 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1855 * corresponding termination condition according to:
1856 * - if REPE/REPZ and ZF = 0 then done
1857 * - if REPNE/REPNZ and ZF = 1 then done
1859 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1860 (c->b == 0xae) || (c->b == 0xaf)) {
1861 if ((c->rep_prefix == REPE_PREFIX) &&
1862 ((ctxt->eflags & EFLG_ZF) == 0)) {
1863 kvm_rip_write(ctxt->vcpu, c->eip);
1866 if ((c->rep_prefix == REPNE_PREFIX) &&
1867 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1868 kvm_rip_write(ctxt->vcpu, c->eip);
1872 c->regs[VCPU_REGS_RCX]--;
1873 c->eip = kvm_rip_read(ctxt->vcpu);
1876 if (c->src.type == OP_MEM) {
1877 c->src.ptr = (unsigned long *)memop;
1879 rc = ops->read_emulated((unsigned long)c->src.ptr,
1883 if (rc != X86EMUL_CONTINUE)
1885 c->src.orig_val = c->src.val;
1888 if ((c->d & DstMask) == ImplicitOps)
1892 if (c->dst.type == OP_MEM) {
1893 c->dst.ptr = (unsigned long *)memop;
1894 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1897 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1899 c->dst.ptr = (void *)c->dst.ptr +
1900 (c->src.val & mask) / 8;
1902 if (!(c->d & Mov)) {
1903 /* optimisation - avoid slow emulated read */
1904 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1908 if (rc != X86EMUL_CONTINUE)
1912 c->dst.orig_val = c->dst.val;
1922 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1924 case 0x06: /* push es */
1925 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1927 case 0x07: /* pop es */
1928 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1934 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1936 case 0x0e: /* push cs */
1937 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1941 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1943 case 0x16: /* push ss */
1944 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1946 case 0x17: /* pop ss */
1947 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1953 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1955 case 0x1e: /* push ds */
1956 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1958 case 0x1f: /* pop ds */
1959 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1965 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1969 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1973 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1977 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1979 case 0x40 ... 0x47: /* inc r16/r32 */
1980 emulate_1op("inc", c->dst, ctxt->eflags);
1982 case 0x48 ... 0x4f: /* dec r16/r32 */
1983 emulate_1op("dec", c->dst, ctxt->eflags);
1985 case 0x50 ... 0x57: /* push reg */
1988 case 0x58 ... 0x5f: /* pop reg */
1990 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1994 case 0x60: /* pusha */
1995 emulate_pusha(ctxt);
1997 case 0x61: /* popa */
1998 rc = emulate_popa(ctxt, ops);
2002 case 0x63: /* movsxd */
2003 if (ctxt->mode != X86EMUL_MODE_PROT64)
2004 goto cannot_emulate;
2005 c->dst.val = (s32) c->src.val;
2007 case 0x68: /* push imm */
2008 case 0x6a: /* push imm8 */
2011 case 0x6c: /* insb */
2012 case 0x6d: /* insw/insd */
2013 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2014 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2015 kvm_inject_gp(ctxt->vcpu, 0);
2018 if (kvm_emulate_pio_string(ctxt->vcpu,
2020 (c->d & ByteOp) ? 1 : c->op_bytes,
2022 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2023 (ctxt->eflags & EFLG_DF),
2024 register_address(c, es_base(ctxt),
2025 c->regs[VCPU_REGS_RDI]),
2027 c->regs[VCPU_REGS_RDX]) == 0) {
2032 case 0x6e: /* outsb */
2033 case 0x6f: /* outsw/outsd */
2034 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2035 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2036 kvm_inject_gp(ctxt->vcpu, 0);
2039 if (kvm_emulate_pio_string(ctxt->vcpu,
2041 (c->d & ByteOp) ? 1 : c->op_bytes,
2043 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2044 (ctxt->eflags & EFLG_DF),
2046 seg_override_base(ctxt, c),
2047 c->regs[VCPU_REGS_RSI]),
2049 c->regs[VCPU_REGS_RDX]) == 0) {
2054 case 0x70 ... 0x7f: /* jcc (short) */
2055 if (test_cc(c->b, ctxt->eflags))
2056 jmp_rel(c, c->src.val);
2058 case 0x80 ... 0x83: /* Grp1 */
2059 switch (c->modrm_reg) {
2079 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2081 case 0x86 ... 0x87: /* xchg */
2083 /* Write back the register source. */
2084 switch (c->dst.bytes) {
2086 *(u8 *) c->src.ptr = (u8) c->dst.val;
2089 *(u16 *) c->src.ptr = (u16) c->dst.val;
2092 *c->src.ptr = (u32) c->dst.val;
2093 break; /* 64b reg: zero-extend */
2095 *c->src.ptr = c->dst.val;
2099 * Write back the memory destination with implicit LOCK
2102 c->dst.val = c->src.val;
2105 case 0x88 ... 0x8b: /* mov */
2107 case 0x8c: { /* mov r/m, sreg */
2108 struct kvm_segment segreg;
2110 if (c->modrm_reg <= 5)
2111 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
2113 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
2115 goto cannot_emulate;
2117 c->dst.val = segreg.selector;
2120 case 0x8d: /* lea r16/r32, m */
2121 c->dst.val = c->modrm_ea;
2123 case 0x8e: { /* mov seg, r/m16 */
2128 if (c->modrm_reg == VCPU_SREG_CS ||
2129 c->modrm_reg > VCPU_SREG_GS) {
2130 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
2134 if (c->modrm_reg == VCPU_SREG_SS)
2135 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
2137 rc = kvm_load_segment_descriptor(ctxt->vcpu, sel, c->modrm_reg);
2139 c->dst.type = OP_NONE; /* Disable writeback. */
2142 case 0x8f: /* pop (sole member of Grp1a) */
2143 rc = emulate_grp1a(ctxt, ops);
2147 case 0x90: /* nop / xchg r8,rax */
2148 if (!(c->rex_prefix & 1)) { /* nop */
2149 c->dst.type = OP_NONE;
2152 case 0x91 ... 0x97: /* xchg reg,rax */
2153 c->src.type = c->dst.type = OP_REG;
2154 c->src.bytes = c->dst.bytes = c->op_bytes;
2155 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2156 c->src.val = *(c->src.ptr);
2158 case 0x9c: /* pushf */
2159 c->src.val = (unsigned long) ctxt->eflags;
2162 case 0x9d: /* popf */
2163 c->dst.type = OP_REG;
2164 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2165 c->dst.bytes = c->op_bytes;
2166 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2167 if (rc != X86EMUL_CONTINUE)
2170 case 0xa0 ... 0xa1: /* mov */
2171 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2172 c->dst.val = c->src.val;
2174 case 0xa2 ... 0xa3: /* mov */
2175 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2177 case 0xa4 ... 0xa5: /* movs */
2178 c->dst.type = OP_MEM;
2179 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2180 c->dst.ptr = (unsigned long *)register_address(c,
2182 c->regs[VCPU_REGS_RDI]);
2183 rc = ops->read_emulated(register_address(c,
2184 seg_override_base(ctxt, c),
2185 c->regs[VCPU_REGS_RSI]),
2187 c->dst.bytes, ctxt->vcpu);
2188 if (rc != X86EMUL_CONTINUE)
2190 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2191 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2193 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2194 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2197 case 0xa6 ... 0xa7: /* cmps */
2198 c->src.type = OP_NONE; /* Disable writeback. */
2199 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2200 c->src.ptr = (unsigned long *)register_address(c,
2201 seg_override_base(ctxt, c),
2202 c->regs[VCPU_REGS_RSI]);
2203 rc = ops->read_emulated((unsigned long)c->src.ptr,
2207 if (rc != X86EMUL_CONTINUE)
2210 c->dst.type = OP_NONE; /* Disable writeback. */
2211 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2212 c->dst.ptr = (unsigned long *)register_address(c,
2214 c->regs[VCPU_REGS_RDI]);
2215 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2219 if (rc != X86EMUL_CONTINUE)
2222 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2224 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2226 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2227 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2229 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2230 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2234 case 0xaa ... 0xab: /* stos */
2235 c->dst.type = OP_MEM;
2236 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2237 c->dst.ptr = (unsigned long *)register_address(c,
2239 c->regs[VCPU_REGS_RDI]);
2240 c->dst.val = c->regs[VCPU_REGS_RAX];
2241 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2242 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2245 case 0xac ... 0xad: /* lods */
2246 c->dst.type = OP_REG;
2247 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2248 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2249 rc = ops->read_emulated(register_address(c,
2250 seg_override_base(ctxt, c),
2251 c->regs[VCPU_REGS_RSI]),
2255 if (rc != X86EMUL_CONTINUE)
2257 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2258 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2261 case 0xae ... 0xaf: /* scas */
2262 DPRINTF("Urk! I don't handle SCAS.\n");
2263 goto cannot_emulate;
2264 case 0xb0 ... 0xbf: /* mov r, imm */
2269 case 0xc3: /* ret */
2270 c->dst.type = OP_REG;
2271 c->dst.ptr = &c->eip;
2272 c->dst.bytes = c->op_bytes;
2273 goto pop_instruction;
2274 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2276 c->dst.val = c->src.val;
2278 case 0xcb: /* ret far */
2279 rc = emulate_ret_far(ctxt, ops);
2283 case 0xd0 ... 0xd1: /* Grp2 */
2287 case 0xd2 ... 0xd3: /* Grp2 */
2288 c->src.val = c->regs[VCPU_REGS_RCX];
2291 case 0xe4: /* inb */
2296 case 0xe6: /* outb */
2297 case 0xe7: /* out */
2301 case 0xe8: /* call (near) */ {
2302 long int rel = c->src.val;
2303 c->src.val = (unsigned long) c->eip;
2308 case 0xe9: /* jmp rel */
2310 case 0xea: /* jmp far */
2311 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val,
2315 c->eip = c->src.val;
2318 jmp: /* jmp rel short */
2319 jmp_rel(c, c->src.val);
2320 c->dst.type = OP_NONE; /* Disable writeback. */
2322 case 0xec: /* in al,dx */
2323 case 0xed: /* in (e/r)ax,dx */
2324 port = c->regs[VCPU_REGS_RDX];
2327 case 0xee: /* out al,dx */
2328 case 0xef: /* out (e/r)ax,dx */
2329 port = c->regs[VCPU_REGS_RDX];
2332 if (!emulator_io_permited(ctxt, ops, port,
2333 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2334 kvm_inject_gp(ctxt->vcpu, 0);
2337 if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2338 (c->d & ByteOp) ? 1 : c->op_bytes,
2341 goto cannot_emulate;
2344 case 0xf4: /* hlt */
2345 ctxt->vcpu->arch.halt_request = 1;
2347 case 0xf5: /* cmc */
2348 /* complement carry flag from eflags reg */
2349 ctxt->eflags ^= EFLG_CF;
2350 c->dst.type = OP_NONE; /* Disable writeback. */
2352 case 0xf6 ... 0xf7: /* Grp3 */
2353 rc = emulate_grp3(ctxt, ops);
2357 case 0xf8: /* clc */
2358 ctxt->eflags &= ~EFLG_CF;
2359 c->dst.type = OP_NONE; /* Disable writeback. */
2361 case 0xfa: /* cli */
2362 if (emulator_bad_iopl(ctxt))
2363 kvm_inject_gp(ctxt->vcpu, 0);
2365 ctxt->eflags &= ~X86_EFLAGS_IF;
2366 c->dst.type = OP_NONE; /* Disable writeback. */
2369 case 0xfb: /* sti */
2370 if (emulator_bad_iopl(ctxt))
2371 kvm_inject_gp(ctxt->vcpu, 0);
2373 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2374 ctxt->eflags |= X86_EFLAGS_IF;
2375 c->dst.type = OP_NONE; /* Disable writeback. */
2378 case 0xfc: /* cld */
2379 ctxt->eflags &= ~EFLG_DF;
2380 c->dst.type = OP_NONE; /* Disable writeback. */
2382 case 0xfd: /* std */
2383 ctxt->eflags |= EFLG_DF;
2384 c->dst.type = OP_NONE; /* Disable writeback. */
2386 case 0xfe ... 0xff: /* Grp4/Grp5 */
2387 rc = emulate_grp45(ctxt, ops);
2394 rc = writeback(ctxt, ops);
2398 /* Commit shadow register state. */
2399 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2400 kvm_rip_write(ctxt->vcpu, c->eip);
2403 if (rc == X86EMUL_UNHANDLEABLE) {
2411 case 0x01: /* lgdt, lidt, lmsw */
2412 switch (c->modrm_reg) {
2414 unsigned long address;
2416 case 0: /* vmcall */
2417 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2418 goto cannot_emulate;
2420 rc = kvm_fix_hypercall(ctxt->vcpu);
2424 /* Let the processor re-execute the fixed hypercall */
2425 c->eip = kvm_rip_read(ctxt->vcpu);
2426 /* Disable writeback. */
2427 c->dst.type = OP_NONE;
2430 rc = read_descriptor(ctxt, ops, c->src.ptr,
2431 &size, &address, c->op_bytes);
2434 realmode_lgdt(ctxt->vcpu, size, address);
2435 /* Disable writeback. */
2436 c->dst.type = OP_NONE;
2438 case 3: /* lidt/vmmcall */
2439 if (c->modrm_mod == 3) {
2440 switch (c->modrm_rm) {
2442 rc = kvm_fix_hypercall(ctxt->vcpu);
2447 goto cannot_emulate;
2450 rc = read_descriptor(ctxt, ops, c->src.ptr,
2455 realmode_lidt(ctxt->vcpu, size, address);
2457 /* Disable writeback. */
2458 c->dst.type = OP_NONE;
2462 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2465 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2467 c->dst.type = OP_NONE;
2470 emulate_invlpg(ctxt->vcpu, memop);
2471 /* Disable writeback. */
2472 c->dst.type = OP_NONE;
2475 goto cannot_emulate;
2478 case 0x05: /* syscall */
2479 rc = emulate_syscall(ctxt);
2480 if (rc != X86EMUL_CONTINUE)
2486 emulate_clts(ctxt->vcpu);
2487 c->dst.type = OP_NONE;
2489 case 0x08: /* invd */
2490 case 0x09: /* wbinvd */
2491 case 0x0d: /* GrpP (prefetch) */
2492 case 0x18: /* Grp16 (prefetch/nop) */
2493 c->dst.type = OP_NONE;
2495 case 0x20: /* mov cr, reg */
2496 if (c->modrm_mod != 3)
2497 goto cannot_emulate;
2498 c->regs[c->modrm_rm] =
2499 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2500 c->dst.type = OP_NONE; /* no writeback */
2502 case 0x21: /* mov from dr to reg */
2503 if (c->modrm_mod != 3)
2504 goto cannot_emulate;
2505 rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
2507 goto cannot_emulate;
2508 c->dst.type = OP_NONE; /* no writeback */
2510 case 0x22: /* mov reg, cr */
2511 if (c->modrm_mod != 3)
2512 goto cannot_emulate;
2513 realmode_set_cr(ctxt->vcpu,
2514 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2515 c->dst.type = OP_NONE;
2517 case 0x23: /* mov from reg to dr */
2518 if (c->modrm_mod != 3)
2519 goto cannot_emulate;
2520 rc = emulator_set_dr(ctxt, c->modrm_reg,
2521 c->regs[c->modrm_rm]);
2523 goto cannot_emulate;
2524 c->dst.type = OP_NONE; /* no writeback */
2528 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2529 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2530 rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
2532 kvm_inject_gp(ctxt->vcpu, 0);
2533 c->eip = kvm_rip_read(ctxt->vcpu);
2535 rc = X86EMUL_CONTINUE;
2536 c->dst.type = OP_NONE;
2540 rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
2542 kvm_inject_gp(ctxt->vcpu, 0);
2543 c->eip = kvm_rip_read(ctxt->vcpu);
2545 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2546 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2548 rc = X86EMUL_CONTINUE;
2549 c->dst.type = OP_NONE;
2551 case 0x34: /* sysenter */
2552 rc = emulate_sysenter(ctxt);
2553 if (rc != X86EMUL_CONTINUE)
2558 case 0x35: /* sysexit */
2559 rc = emulate_sysexit(ctxt);
2560 if (rc != X86EMUL_CONTINUE)
2565 case 0x40 ... 0x4f: /* cmov */
2566 c->dst.val = c->dst.orig_val = c->src.val;
2567 if (!test_cc(c->b, ctxt->eflags))
2568 c->dst.type = OP_NONE; /* no writeback */
2570 case 0x80 ... 0x8f: /* jnz rel, etc*/
2571 if (test_cc(c->b, ctxt->eflags))
2572 jmp_rel(c, c->src.val);
2573 c->dst.type = OP_NONE;
2575 case 0xa0: /* push fs */
2576 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2578 case 0xa1: /* pop fs */
2579 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2585 c->dst.type = OP_NONE;
2586 /* only subword offset */
2587 c->src.val &= (c->dst.bytes << 3) - 1;
2588 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2590 case 0xa4: /* shld imm8, r, r/m */
2591 case 0xa5: /* shld cl, r, r/m */
2592 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2594 case 0xa8: /* push gs */
2595 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2597 case 0xa9: /* pop gs */
2598 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2604 /* only subword offset */
2605 c->src.val &= (c->dst.bytes << 3) - 1;
2606 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2608 case 0xac: /* shrd imm8, r, r/m */
2609 case 0xad: /* shrd cl, r, r/m */
2610 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2612 case 0xae: /* clflush */
2614 case 0xb0 ... 0xb1: /* cmpxchg */
2616 * Save real source value, then compare EAX against
2619 c->src.orig_val = c->src.val;
2620 c->src.val = c->regs[VCPU_REGS_RAX];
2621 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2622 if (ctxt->eflags & EFLG_ZF) {
2623 /* Success: write back to memory. */
2624 c->dst.val = c->src.orig_val;
2626 /* Failure: write the value we saw to EAX. */
2627 c->dst.type = OP_REG;
2628 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2633 /* only subword offset */
2634 c->src.val &= (c->dst.bytes << 3) - 1;
2635 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2637 case 0xb6 ... 0xb7: /* movzx */
2638 c->dst.bytes = c->op_bytes;
2639 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2642 case 0xba: /* Grp8 */
2643 switch (c->modrm_reg & 3) {
2656 /* only subword offset */
2657 c->src.val &= (c->dst.bytes << 3) - 1;
2658 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2660 case 0xbe ... 0xbf: /* movsx */
2661 c->dst.bytes = c->op_bytes;
2662 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2665 case 0xc3: /* movnti */
2666 c->dst.bytes = c->op_bytes;
2667 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2670 case 0xc7: /* Grp9 (cmpxchg8b) */
2671 rc = emulate_grp9(ctxt, ops, memop);
2674 c->dst.type = OP_NONE;
2680 DPRINTF("Cannot emulate %02x\n", c->b);