1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
79 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
80 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
82 /* Source 2 operand type */
83 #define Src2None (0<<29)
84 #define Src2CL (1<<29)
85 #define Src2ImmByte (2<<29)
86 #define Src2One (3<<29)
87 #define Src2Imm16 (4<<29)
88 #define Src2Mem16 (5<<29) /* Used for Ep encoding. First argument has to be
89 in memory and second argument is located
90 immediately after the first one in memory. */
91 #define Src2Mask (7<<29)
94 Group1_80, Group1_81, Group1_82, Group1_83,
95 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
99 static u32 opcode_table[256] = {
101 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
102 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
103 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
104 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
106 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
107 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
108 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
109 ImplicitOps | Stack | No64, 0,
111 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
112 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
113 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
114 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
116 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
117 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
118 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
119 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
121 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
122 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
123 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
125 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
126 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
129 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
130 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
133 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
134 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
135 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
138 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
140 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
142 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
143 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
145 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
146 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
148 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
149 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
152 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
153 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
154 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
156 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
159 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
160 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
162 Group | Group1_80, Group | Group1_81,
163 Group | Group1_82, Group | Group1_83,
164 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
165 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
167 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
168 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
169 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
170 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
172 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
174 0, 0, SrcImm | Src2Imm16 | No64, 0,
175 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
177 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
178 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
179 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
180 ByteOp | ImplicitOps | String, ImplicitOps | String,
182 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
183 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
184 ByteOp | ImplicitOps | String, ImplicitOps | String,
186 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
187 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
188 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
189 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
191 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
192 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
193 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
194 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
196 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
197 0, ImplicitOps | Stack, 0, 0,
198 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
200 0, 0, 0, ImplicitOps | Stack,
201 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
203 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
204 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
207 0, 0, 0, 0, 0, 0, 0, 0,
210 ByteOp | SrcImmUByte, SrcImmUByte,
211 ByteOp | SrcImmUByte, SrcImmUByte,
213 SrcImm | Stack, SrcImm | ImplicitOps,
214 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
215 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
216 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
219 ImplicitOps | Priv, ImplicitOps, Group | Group3_Byte, Group | Group3,
221 ImplicitOps, 0, ImplicitOps, ImplicitOps,
222 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
225 static u32 twobyte_table[256] = {
227 0, Group | GroupDual | Group7, 0, 0,
228 0, ImplicitOps, ImplicitOps | Priv, 0,
229 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
230 0, ImplicitOps | ModRM, 0, 0,
232 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
234 ModRM | ImplicitOps | Priv, ModRM | Priv,
235 ModRM | ImplicitOps | Priv, ModRM | Priv,
237 0, 0, 0, 0, 0, 0, 0, 0,
239 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
240 ImplicitOps, ImplicitOps | Priv, 0, 0,
241 0, 0, 0, 0, 0, 0, 0, 0,
243 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
244 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
245 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
246 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
248 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
249 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
250 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
251 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
253 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
255 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
257 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
259 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
260 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
262 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
264 ImplicitOps | Stack, ImplicitOps | Stack,
265 0, DstMem | SrcReg | ModRM | BitOp,
266 DstMem | SrcReg | Src2ImmByte | ModRM,
267 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
269 ImplicitOps | Stack, ImplicitOps | Stack,
270 0, DstMem | SrcReg | ModRM | BitOp | Lock,
271 DstMem | SrcReg | Src2ImmByte | ModRM,
272 DstMem | SrcReg | Src2CL | ModRM,
275 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
276 0, DstMem | SrcReg | ModRM | BitOp | Lock,
277 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
278 DstReg | SrcMem16 | ModRM | Mov,
281 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
282 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
283 DstReg | SrcMem16 | ModRM | Mov,
285 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
286 0, 0, 0, Group | GroupDual | Group9,
287 0, 0, 0, 0, 0, 0, 0, 0,
289 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
291 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
293 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
296 static u32 group_table[] = {
298 ByteOp | DstMem | SrcImm | ModRM | Lock,
299 ByteOp | DstMem | SrcImm | ModRM | Lock,
300 ByteOp | DstMem | SrcImm | ModRM | Lock,
301 ByteOp | DstMem | SrcImm | ModRM | Lock,
302 ByteOp | DstMem | SrcImm | ModRM | Lock,
303 ByteOp | DstMem | SrcImm | ModRM | Lock,
304 ByteOp | DstMem | SrcImm | ModRM | Lock,
305 ByteOp | DstMem | SrcImm | ModRM,
307 DstMem | SrcImm | ModRM | Lock,
308 DstMem | SrcImm | ModRM | Lock,
309 DstMem | SrcImm | ModRM | Lock,
310 DstMem | SrcImm | ModRM | Lock,
311 DstMem | SrcImm | ModRM | Lock,
312 DstMem | SrcImm | ModRM | Lock,
313 DstMem | SrcImm | ModRM | Lock,
314 DstMem | SrcImm | ModRM,
316 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
317 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
318 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
319 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
320 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
321 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
322 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
323 ByteOp | DstMem | SrcImm | ModRM | No64,
325 DstMem | SrcImmByte | ModRM | Lock,
326 DstMem | SrcImmByte | ModRM | Lock,
327 DstMem | SrcImmByte | ModRM | Lock,
328 DstMem | SrcImmByte | ModRM | Lock,
329 DstMem | SrcImmByte | ModRM | Lock,
330 DstMem | SrcImmByte | ModRM | Lock,
331 DstMem | SrcImmByte | ModRM | Lock,
332 DstMem | SrcImmByte | ModRM,
334 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
336 ByteOp | SrcImm | DstMem | ModRM, 0,
337 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
340 DstMem | SrcImm | ModRM, 0,
341 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
344 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
347 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
348 SrcMem | ModRM | Stack, 0,
349 SrcMem | ModRM | Stack, SrcMem | ModRM | Src2Mem16 | ImplicitOps,
350 SrcMem | ModRM | Stack, 0,
352 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
353 SrcNone | ModRM | DstMem | Mov, 0,
354 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
357 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
358 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
360 0, ImplicitOps | ModRM | Lock, 0, 0, 0, 0, 0, 0,
363 static u32 group2_table[] = {
365 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM | Priv,
366 SrcNone | ModRM | DstMem | Mov, 0,
367 SrcMem16 | ModRM | Mov | Priv, 0,
369 0, 0, 0, 0, 0, 0, 0, 0,
372 /* EFLAGS bit definitions. */
373 #define EFLG_ID (1<<21)
374 #define EFLG_VIP (1<<20)
375 #define EFLG_VIF (1<<19)
376 #define EFLG_AC (1<<18)
377 #define EFLG_VM (1<<17)
378 #define EFLG_RF (1<<16)
379 #define EFLG_IOPL (3<<12)
380 #define EFLG_NT (1<<14)
381 #define EFLG_OF (1<<11)
382 #define EFLG_DF (1<<10)
383 #define EFLG_IF (1<<9)
384 #define EFLG_TF (1<<8)
385 #define EFLG_SF (1<<7)
386 #define EFLG_ZF (1<<6)
387 #define EFLG_AF (1<<4)
388 #define EFLG_PF (1<<2)
389 #define EFLG_CF (1<<0)
392 * Instruction emulation:
393 * Most instructions are emulated directly via a fragment of inline assembly
394 * code. This allows us to save/restore EFLAGS and thus very easily pick up
395 * any modified flags.
398 #if defined(CONFIG_X86_64)
399 #define _LO32 "k" /* force 32-bit operand */
400 #define _STK "%%rsp" /* stack pointer */
401 #elif defined(__i386__)
402 #define _LO32 "" /* force 32-bit operand */
403 #define _STK "%%esp" /* stack pointer */
407 * These EFLAGS bits are restored from saved value during emulation, and
408 * any changes are written back to the saved value after emulation.
410 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
412 /* Before executing instruction: restore necessary bits in EFLAGS. */
413 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
414 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
415 "movl %"_sav",%"_LO32 _tmp"; " \
418 "movl %"_msk",%"_LO32 _tmp"; " \
419 "andl %"_LO32 _tmp",("_STK"); " \
421 "notl %"_LO32 _tmp"; " \
422 "andl %"_LO32 _tmp",("_STK"); " \
423 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
425 "orl %"_LO32 _tmp",("_STK"); " \
429 /* After executing instruction: write-back necessary bits in EFLAGS. */
430 #define _POST_EFLAGS(_sav, _msk, _tmp) \
431 /* _sav |= EFLAGS & _msk; */ \
434 "andl %"_msk",%"_LO32 _tmp"; " \
435 "orl %"_LO32 _tmp",%"_sav"; "
443 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
445 __asm__ __volatile__ ( \
446 _PRE_EFLAGS("0", "4", "2") \
447 _op _suffix " %"_x"3,%1; " \
448 _POST_EFLAGS("0", "4", "2") \
449 : "=m" (_eflags), "=m" ((_dst).val), \
451 : _y ((_src).val), "i" (EFLAGS_MASK)); \
455 /* Raw emulation: instruction has two explicit operands. */
456 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
458 unsigned long _tmp; \
460 switch ((_dst).bytes) { \
462 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
465 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
468 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
473 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
475 unsigned long _tmp; \
476 switch ((_dst).bytes) { \
478 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
481 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
482 _wx, _wy, _lx, _ly, _qx, _qy); \
487 /* Source operand is byte-sized and may be restricted to just %cl. */
488 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
489 __emulate_2op(_op, _src, _dst, _eflags, \
490 "b", "c", "b", "c", "b", "c", "b", "c")
492 /* Source operand is byte, word, long or quad sized. */
493 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
494 __emulate_2op(_op, _src, _dst, _eflags, \
495 "b", "q", "w", "r", _LO32, "r", "", "r")
497 /* Source operand is word, long or quad sized. */
498 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
499 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
500 "w", "r", _LO32, "r", "", "r")
502 /* Instruction has three operands and one operand is stored in ECX register */
503 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
505 unsigned long _tmp; \
506 _type _clv = (_cl).val; \
507 _type _srcv = (_src).val; \
508 _type _dstv = (_dst).val; \
510 __asm__ __volatile__ ( \
511 _PRE_EFLAGS("0", "5", "2") \
512 _op _suffix " %4,%1 \n" \
513 _POST_EFLAGS("0", "5", "2") \
514 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
515 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
518 (_cl).val = (unsigned long) _clv; \
519 (_src).val = (unsigned long) _srcv; \
520 (_dst).val = (unsigned long) _dstv; \
523 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
525 switch ((_dst).bytes) { \
527 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
528 "w", unsigned short); \
531 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
532 "l", unsigned int); \
535 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
536 "q", unsigned long)); \
541 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
543 unsigned long _tmp; \
545 __asm__ __volatile__ ( \
546 _PRE_EFLAGS("0", "3", "2") \
547 _op _suffix " %1; " \
548 _POST_EFLAGS("0", "3", "2") \
549 : "=m" (_eflags), "+m" ((_dst).val), \
551 : "i" (EFLAGS_MASK)); \
554 /* Instruction has only one explicit operand (no source operand). */
555 #define emulate_1op(_op, _dst, _eflags) \
557 switch ((_dst).bytes) { \
558 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
559 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
560 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
561 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
565 /* Fetch next part of the instruction being emulated. */
566 #define insn_fetch(_type, _size, _eip) \
567 ({ unsigned long _x; \
568 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
575 static inline unsigned long ad_mask(struct decode_cache *c)
577 return (1UL << (c->ad_bytes << 3)) - 1;
580 /* Access/update address held in a register, based on addressing mode. */
581 static inline unsigned long
582 address_mask(struct decode_cache *c, unsigned long reg)
584 if (c->ad_bytes == sizeof(unsigned long))
587 return reg & ad_mask(c);
590 static inline unsigned long
591 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
593 return base + address_mask(c, reg);
597 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
599 if (c->ad_bytes == sizeof(unsigned long))
602 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
605 static inline void jmp_rel(struct decode_cache *c, int rel)
607 register_address_increment(c, &c->eip, rel);
610 static void set_seg_override(struct decode_cache *c, int seg)
612 c->has_seg_override = true;
613 c->seg_override = seg;
616 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
618 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
621 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
624 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
625 struct decode_cache *c)
627 if (!c->has_seg_override)
630 return seg_base(ctxt, c->seg_override);
633 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
635 return seg_base(ctxt, VCPU_SREG_ES);
638 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
640 return seg_base(ctxt, VCPU_SREG_SS);
643 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
644 struct x86_emulate_ops *ops,
645 unsigned long linear, u8 *dest)
647 struct fetch_cache *fc = &ctxt->decode.fetch;
651 if (linear < fc->start || linear >= fc->end) {
652 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
653 rc = ops->fetch(linear, fc->data, size, ctxt->vcpu, NULL);
654 if (rc != X86EMUL_CONTINUE)
657 fc->end = linear + size;
659 *dest = fc->data[linear - fc->start];
660 return X86EMUL_CONTINUE;
663 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
664 struct x86_emulate_ops *ops,
665 unsigned long eip, void *dest, unsigned size)
669 /* x86 instructions are limited to 15 bytes. */
670 if (eip + size - ctxt->decode.eip_orig > 15)
671 return X86EMUL_UNHANDLEABLE;
672 eip += ctxt->cs_base;
674 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
675 if (rc != X86EMUL_CONTINUE)
678 return X86EMUL_CONTINUE;
682 * Given the 'reg' portion of a ModRM byte, and a register block, return a
683 * pointer into the block that addresses the relevant register.
684 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
686 static void *decode_register(u8 modrm_reg, unsigned long *regs,
691 p = ®s[modrm_reg];
692 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
693 p = (unsigned char *)®s[modrm_reg & 3] + 1;
697 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
698 struct x86_emulate_ops *ops,
700 u16 *size, unsigned long *address, int op_bytes)
707 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
709 if (rc != X86EMUL_CONTINUE)
711 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
716 static int test_cc(unsigned int condition, unsigned int flags)
720 switch ((condition & 15) >> 1) {
722 rc |= (flags & EFLG_OF);
724 case 1: /* b/c/nae */
725 rc |= (flags & EFLG_CF);
728 rc |= (flags & EFLG_ZF);
731 rc |= (flags & (EFLG_CF|EFLG_ZF));
734 rc |= (flags & EFLG_SF);
737 rc |= (flags & EFLG_PF);
740 rc |= (flags & EFLG_ZF);
743 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
747 /* Odd condition identifiers (lsb == 1) have inverted sense. */
748 return (!!rc ^ (condition & 1));
751 static void decode_register_operand(struct operand *op,
752 struct decode_cache *c,
755 unsigned reg = c->modrm_reg;
756 int highbyte_regs = c->rex_prefix == 0;
759 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
761 if ((c->d & ByteOp) && !inhibit_bytereg) {
762 op->ptr = decode_register(reg, c->regs, highbyte_regs);
763 op->val = *(u8 *)op->ptr;
766 op->ptr = decode_register(reg, c->regs, 0);
767 op->bytes = c->op_bytes;
770 op->val = *(u16 *)op->ptr;
773 op->val = *(u32 *)op->ptr;
776 op->val = *(u64 *) op->ptr;
780 op->orig_val = op->val;
783 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
784 struct x86_emulate_ops *ops)
786 struct decode_cache *c = &ctxt->decode;
788 int index_reg = 0, base_reg = 0, scale;
789 int rc = X86EMUL_CONTINUE;
792 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
793 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
794 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
797 c->modrm = insn_fetch(u8, 1, c->eip);
798 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
799 c->modrm_reg |= (c->modrm & 0x38) >> 3;
800 c->modrm_rm |= (c->modrm & 0x07);
804 if (c->modrm_mod == 3) {
805 c->modrm_ptr = decode_register(c->modrm_rm,
806 c->regs, c->d & ByteOp);
807 c->modrm_val = *(unsigned long *)c->modrm_ptr;
811 if (c->ad_bytes == 2) {
812 unsigned bx = c->regs[VCPU_REGS_RBX];
813 unsigned bp = c->regs[VCPU_REGS_RBP];
814 unsigned si = c->regs[VCPU_REGS_RSI];
815 unsigned di = c->regs[VCPU_REGS_RDI];
817 /* 16-bit ModR/M decode. */
818 switch (c->modrm_mod) {
820 if (c->modrm_rm == 6)
821 c->modrm_ea += insn_fetch(u16, 2, c->eip);
824 c->modrm_ea += insn_fetch(s8, 1, c->eip);
827 c->modrm_ea += insn_fetch(u16, 2, c->eip);
830 switch (c->modrm_rm) {
832 c->modrm_ea += bx + si;
835 c->modrm_ea += bx + di;
838 c->modrm_ea += bp + si;
841 c->modrm_ea += bp + di;
850 if (c->modrm_mod != 0)
857 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
858 (c->modrm_rm == 6 && c->modrm_mod != 0))
859 if (!c->has_seg_override)
860 set_seg_override(c, VCPU_SREG_SS);
861 c->modrm_ea = (u16)c->modrm_ea;
863 /* 32/64-bit ModR/M decode. */
864 if ((c->modrm_rm & 7) == 4) {
865 sib = insn_fetch(u8, 1, c->eip);
866 index_reg |= (sib >> 3) & 7;
870 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
871 c->modrm_ea += insn_fetch(s32, 4, c->eip);
873 c->modrm_ea += c->regs[base_reg];
875 c->modrm_ea += c->regs[index_reg] << scale;
876 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
877 if (ctxt->mode == X86EMUL_MODE_PROT64)
880 c->modrm_ea += c->regs[c->modrm_rm];
881 switch (c->modrm_mod) {
883 if (c->modrm_rm == 5)
884 c->modrm_ea += insn_fetch(s32, 4, c->eip);
887 c->modrm_ea += insn_fetch(s8, 1, c->eip);
890 c->modrm_ea += insn_fetch(s32, 4, c->eip);
898 static int decode_abs(struct x86_emulate_ctxt *ctxt,
899 struct x86_emulate_ops *ops)
901 struct decode_cache *c = &ctxt->decode;
902 int rc = X86EMUL_CONTINUE;
904 switch (c->ad_bytes) {
906 c->modrm_ea = insn_fetch(u16, 2, c->eip);
909 c->modrm_ea = insn_fetch(u32, 4, c->eip);
912 c->modrm_ea = insn_fetch(u64, 8, c->eip);
920 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
922 struct decode_cache *c = &ctxt->decode;
923 int rc = X86EMUL_CONTINUE;
924 int mode = ctxt->mode;
925 int def_op_bytes, def_ad_bytes, group;
927 /* Shadow copy of register state. Committed on successful emulation. */
929 memset(c, 0, sizeof(struct decode_cache));
930 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
931 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
932 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
935 case X86EMUL_MODE_REAL:
936 case X86EMUL_MODE_VM86:
937 case X86EMUL_MODE_PROT16:
938 def_op_bytes = def_ad_bytes = 2;
940 case X86EMUL_MODE_PROT32:
941 def_op_bytes = def_ad_bytes = 4;
944 case X86EMUL_MODE_PROT64:
953 c->op_bytes = def_op_bytes;
954 c->ad_bytes = def_ad_bytes;
956 /* Legacy prefixes. */
958 switch (c->b = insn_fetch(u8, 1, c->eip)) {
959 case 0x66: /* operand-size override */
960 /* switch between 2/4 bytes */
961 c->op_bytes = def_op_bytes ^ 6;
963 case 0x67: /* address-size override */
964 if (mode == X86EMUL_MODE_PROT64)
965 /* switch between 4/8 bytes */
966 c->ad_bytes = def_ad_bytes ^ 12;
968 /* switch between 2/4 bytes */
969 c->ad_bytes = def_ad_bytes ^ 6;
971 case 0x26: /* ES override */
972 case 0x2e: /* CS override */
973 case 0x36: /* SS override */
974 case 0x3e: /* DS override */
975 set_seg_override(c, (c->b >> 3) & 3);
977 case 0x64: /* FS override */
978 case 0x65: /* GS override */
979 set_seg_override(c, c->b & 7);
981 case 0x40 ... 0x4f: /* REX */
982 if (mode != X86EMUL_MODE_PROT64)
984 c->rex_prefix = c->b;
986 case 0xf0: /* LOCK */
989 case 0xf2: /* REPNE/REPNZ */
990 c->rep_prefix = REPNE_PREFIX;
992 case 0xf3: /* REP/REPE/REPZ */
993 c->rep_prefix = REPE_PREFIX;
999 /* Any legacy prefix after a REX prefix nullifies its effect. */
1008 if (c->rex_prefix & 8)
1009 c->op_bytes = 8; /* REX.W */
1011 /* Opcode byte(s). */
1012 c->d = opcode_table[c->b];
1014 /* Two-byte opcode? */
1017 c->b = insn_fetch(u8, 1, c->eip);
1018 c->d = twobyte_table[c->b];
1023 group = c->d & GroupMask;
1024 c->modrm = insn_fetch(u8, 1, c->eip);
1027 group = (group << 3) + ((c->modrm >> 3) & 7);
1028 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1029 c->d = group2_table[group];
1031 c->d = group_table[group];
1036 DPRINTF("Cannot emulate %02x\n", c->b);
1040 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1043 /* ModRM and SIB bytes. */
1045 rc = decode_modrm(ctxt, ops);
1046 else if (c->d & MemAbs)
1047 rc = decode_abs(ctxt, ops);
1048 if (rc != X86EMUL_CONTINUE)
1051 if (!c->has_seg_override)
1052 set_seg_override(c, VCPU_SREG_DS);
1054 if (!(!c->twobyte && c->b == 0x8d))
1055 c->modrm_ea += seg_override_base(ctxt, c);
1057 if (c->ad_bytes != 8)
1058 c->modrm_ea = (u32)c->modrm_ea;
1060 * Decode and fetch the source operand: register, memory
1063 switch (c->d & SrcMask) {
1067 decode_register_operand(&c->src, c, 0);
1076 c->src.bytes = (c->d & ByteOp) ? 1 :
1078 /* Don't fetch the address for invlpg: it could be unmapped. */
1079 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1083 * For instructions with a ModR/M byte, switch to register
1084 * access if Mod = 3.
1086 if ((c->d & ModRM) && c->modrm_mod == 3) {
1087 c->src.type = OP_REG;
1088 c->src.val = c->modrm_val;
1089 c->src.ptr = c->modrm_ptr;
1092 c->src.type = OP_MEM;
1096 c->src.type = OP_IMM;
1097 c->src.ptr = (unsigned long *)c->eip;
1098 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1099 if (c->src.bytes == 8)
1101 /* NB. Immediates are sign-extended as necessary. */
1102 switch (c->src.bytes) {
1104 c->src.val = insn_fetch(s8, 1, c->eip);
1107 c->src.val = insn_fetch(s16, 2, c->eip);
1110 c->src.val = insn_fetch(s32, 4, c->eip);
1113 if ((c->d & SrcMask) == SrcImmU) {
1114 switch (c->src.bytes) {
1119 c->src.val &= 0xffff;
1122 c->src.val &= 0xffffffff;
1129 c->src.type = OP_IMM;
1130 c->src.ptr = (unsigned long *)c->eip;
1132 if ((c->d & SrcMask) == SrcImmByte)
1133 c->src.val = insn_fetch(s8, 1, c->eip);
1135 c->src.val = insn_fetch(u8, 1, c->eip);
1144 * Decode and fetch the second source operand: register, memory
1147 switch (c->d & Src2Mask) {
1152 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1155 c->src2.type = OP_IMM;
1156 c->src2.ptr = (unsigned long *)c->eip;
1158 c->src2.val = insn_fetch(u8, 1, c->eip);
1161 c->src2.type = OP_IMM;
1162 c->src2.ptr = (unsigned long *)c->eip;
1164 c->src2.val = insn_fetch(u16, 2, c->eip);
1172 c->src2.type = OP_MEM;
1176 /* Decode and fetch the destination operand: register or memory. */
1177 switch (c->d & DstMask) {
1179 /* Special instructions do their own operand decoding. */
1182 decode_register_operand(&c->dst, c,
1183 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1186 if ((c->d & ModRM) && c->modrm_mod == 3) {
1187 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1188 c->dst.type = OP_REG;
1189 c->dst.val = c->dst.orig_val = c->modrm_val;
1190 c->dst.ptr = c->modrm_ptr;
1193 c->dst.type = OP_MEM;
1196 c->dst.type = OP_REG;
1197 c->dst.bytes = c->op_bytes;
1198 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1199 switch (c->op_bytes) {
1201 c->dst.val = *(u8 *)c->dst.ptr;
1204 c->dst.val = *(u16 *)c->dst.ptr;
1207 c->dst.val = *(u32 *)c->dst.ptr;
1210 c->dst.orig_val = c->dst.val;
1214 if (c->rip_relative)
1215 c->modrm_ea += c->eip;
1218 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1221 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1223 struct decode_cache *c = &ctxt->decode;
1225 c->dst.type = OP_MEM;
1226 c->dst.bytes = c->op_bytes;
1227 c->dst.val = c->src.val;
1228 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1229 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1230 c->regs[VCPU_REGS_RSP]);
1233 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1234 struct x86_emulate_ops *ops,
1235 void *dest, int len)
1237 struct decode_cache *c = &ctxt->decode;
1240 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1241 c->regs[VCPU_REGS_RSP]),
1242 dest, len, ctxt->vcpu);
1243 if (rc != X86EMUL_CONTINUE)
1246 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1250 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1251 struct x86_emulate_ops *ops,
1252 void *dest, int len)
1255 unsigned long val, change_mask;
1256 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1257 int cpl = kvm_x86_ops->get_cpl(ctxt->vcpu);
1259 rc = emulate_pop(ctxt, ops, &val, len);
1260 if (rc != X86EMUL_CONTINUE)
1263 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1264 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1266 switch(ctxt->mode) {
1267 case X86EMUL_MODE_PROT64:
1268 case X86EMUL_MODE_PROT32:
1269 case X86EMUL_MODE_PROT16:
1271 change_mask |= EFLG_IOPL;
1273 change_mask |= EFLG_IF;
1275 case X86EMUL_MODE_VM86:
1277 kvm_inject_gp(ctxt->vcpu, 0);
1278 return X86EMUL_PROPAGATE_FAULT;
1280 change_mask |= EFLG_IF;
1282 default: /* real mode */
1283 change_mask |= (EFLG_IOPL | EFLG_IF);
1287 *(unsigned long *)dest =
1288 (ctxt->eflags & ~change_mask) | (val & change_mask);
1293 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1295 struct decode_cache *c = &ctxt->decode;
1296 struct kvm_segment segment;
1298 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1300 c->src.val = segment.selector;
1304 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1305 struct x86_emulate_ops *ops, int seg)
1307 struct decode_cache *c = &ctxt->decode;
1308 unsigned long selector;
1311 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1312 if (rc != X86EMUL_CONTINUE)
1315 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, seg);
1319 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1321 struct decode_cache *c = &ctxt->decode;
1322 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1323 int reg = VCPU_REGS_RAX;
1325 while (reg <= VCPU_REGS_RDI) {
1326 (reg == VCPU_REGS_RSP) ?
1327 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1334 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1335 struct x86_emulate_ops *ops)
1337 struct decode_cache *c = &ctxt->decode;
1338 int rc = X86EMUL_CONTINUE;
1339 int reg = VCPU_REGS_RDI;
1341 while (reg >= VCPU_REGS_RAX) {
1342 if (reg == VCPU_REGS_RSP) {
1343 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1348 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1349 if (rc != X86EMUL_CONTINUE)
1356 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1357 struct x86_emulate_ops *ops)
1359 struct decode_cache *c = &ctxt->decode;
1361 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1364 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1366 struct decode_cache *c = &ctxt->decode;
1367 switch (c->modrm_reg) {
1369 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1372 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1375 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1378 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1380 case 4: /* sal/shl */
1381 case 6: /* sal/shl */
1382 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1385 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1388 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1393 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1394 struct x86_emulate_ops *ops)
1396 struct decode_cache *c = &ctxt->decode;
1397 int rc = X86EMUL_CONTINUE;
1399 switch (c->modrm_reg) {
1400 case 0 ... 1: /* test */
1401 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1404 c->dst.val = ~c->dst.val;
1407 emulate_1op("neg", c->dst, ctxt->eflags);
1410 DPRINTF("Cannot emulate %02x\n", c->b);
1411 rc = X86EMUL_UNHANDLEABLE;
1417 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1418 struct x86_emulate_ops *ops)
1420 struct decode_cache *c = &ctxt->decode;
1422 switch (c->modrm_reg) {
1424 emulate_1op("inc", c->dst, ctxt->eflags);
1427 emulate_1op("dec", c->dst, ctxt->eflags);
1429 case 2: /* call near abs */ {
1432 c->eip = c->src.val;
1433 c->src.val = old_eip;
1437 case 4: /* jmp abs */
1438 c->eip = c->src.val;
1444 return X86EMUL_CONTINUE;
1447 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1448 struct x86_emulate_ops *ops,
1449 unsigned long memop)
1451 struct decode_cache *c = &ctxt->decode;
1455 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1456 if (rc != X86EMUL_CONTINUE)
1459 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1460 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1462 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1463 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1464 ctxt->eflags &= ~EFLG_ZF;
1467 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1468 (u32) c->regs[VCPU_REGS_RBX];
1470 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1471 if (rc != X86EMUL_CONTINUE)
1473 ctxt->eflags |= EFLG_ZF;
1475 return X86EMUL_CONTINUE;
1478 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1479 struct x86_emulate_ops *ops)
1481 struct decode_cache *c = &ctxt->decode;
1485 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1486 if (rc != X86EMUL_CONTINUE)
1488 if (c->op_bytes == 4)
1489 c->eip = (u32)c->eip;
1490 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1491 if (rc != X86EMUL_CONTINUE)
1493 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, VCPU_SREG_CS);
1497 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1498 struct x86_emulate_ops *ops)
1501 struct decode_cache *c = &ctxt->decode;
1503 switch (c->dst.type) {
1505 /* The 4-byte case *is* correct:
1506 * in 64-bit mode we zero-extend.
1508 switch (c->dst.bytes) {
1510 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1513 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1516 *c->dst.ptr = (u32)c->dst.val;
1517 break; /* 64b: zero-ext */
1519 *c->dst.ptr = c->dst.val;
1525 rc = ops->cmpxchg_emulated(
1526 (unsigned long)c->dst.ptr,
1532 rc = ops->write_emulated(
1533 (unsigned long)c->dst.ptr,
1537 if (rc != X86EMUL_CONTINUE)
1546 return X86EMUL_CONTINUE;
1549 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1551 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1553 * an sti; sti; sequence only disable interrupts for the first
1554 * instruction. So, if the last instruction, be it emulated or
1555 * not, left the system with the INT_STI flag enabled, it
1556 * means that the last instruction is an sti. We should not
1557 * leave the flag on in this case. The same goes for mov ss
1559 if (!(int_shadow & mask))
1560 ctxt->interruptibility = mask;
1564 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1565 struct kvm_segment *cs, struct kvm_segment *ss)
1567 memset(cs, 0, sizeof(struct kvm_segment));
1568 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1569 memset(ss, 0, sizeof(struct kvm_segment));
1571 cs->l = 0; /* will be adjusted later */
1572 cs->base = 0; /* flat segment */
1573 cs->g = 1; /* 4kb granularity */
1574 cs->limit = 0xffffffff; /* 4GB limit */
1575 cs->type = 0x0b; /* Read, Execute, Accessed */
1577 cs->dpl = 0; /* will be adjusted later */
1582 ss->base = 0; /* flat segment */
1583 ss->limit = 0xffffffff; /* 4GB limit */
1584 ss->g = 1; /* 4kb granularity */
1586 ss->type = 0x03; /* Read/Write, Accessed */
1587 ss->db = 1; /* 32bit stack segment */
1593 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1595 struct decode_cache *c = &ctxt->decode;
1596 struct kvm_segment cs, ss;
1599 /* syscall is not available in real mode */
1600 if (ctxt->mode == X86EMUL_MODE_REAL || ctxt->mode == X86EMUL_MODE_VM86)
1601 return X86EMUL_UNHANDLEABLE;
1603 setup_syscalls_segments(ctxt, &cs, &ss);
1605 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1607 cs.selector = (u16)(msr_data & 0xfffc);
1608 ss.selector = (u16)(msr_data + 8);
1610 if (is_long_mode(ctxt->vcpu)) {
1614 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1615 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1617 c->regs[VCPU_REGS_RCX] = c->eip;
1618 if (is_long_mode(ctxt->vcpu)) {
1619 #ifdef CONFIG_X86_64
1620 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1622 kvm_x86_ops->get_msr(ctxt->vcpu,
1623 ctxt->mode == X86EMUL_MODE_PROT64 ?
1624 MSR_LSTAR : MSR_CSTAR, &msr_data);
1627 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1628 ctxt->eflags &= ~(msr_data | EFLG_RF);
1632 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1633 c->eip = (u32)msr_data;
1635 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1638 return X86EMUL_CONTINUE;
1642 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1644 struct decode_cache *c = &ctxt->decode;
1645 struct kvm_segment cs, ss;
1648 /* inject #GP if in real mode */
1649 if (ctxt->mode == X86EMUL_MODE_REAL) {
1650 kvm_inject_gp(ctxt->vcpu, 0);
1651 return X86EMUL_UNHANDLEABLE;
1654 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1655 * Therefore, we inject an #UD.
1657 if (ctxt->mode == X86EMUL_MODE_PROT64)
1658 return X86EMUL_UNHANDLEABLE;
1660 setup_syscalls_segments(ctxt, &cs, &ss);
1662 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1663 switch (ctxt->mode) {
1664 case X86EMUL_MODE_PROT32:
1665 if ((msr_data & 0xfffc) == 0x0) {
1666 kvm_inject_gp(ctxt->vcpu, 0);
1667 return X86EMUL_PROPAGATE_FAULT;
1670 case X86EMUL_MODE_PROT64:
1671 if (msr_data == 0x0) {
1672 kvm_inject_gp(ctxt->vcpu, 0);
1673 return X86EMUL_PROPAGATE_FAULT;
1678 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1679 cs.selector = (u16)msr_data;
1680 cs.selector &= ~SELECTOR_RPL_MASK;
1681 ss.selector = cs.selector + 8;
1682 ss.selector &= ~SELECTOR_RPL_MASK;
1683 if (ctxt->mode == X86EMUL_MODE_PROT64
1684 || is_long_mode(ctxt->vcpu)) {
1689 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1690 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1692 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1695 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1696 c->regs[VCPU_REGS_RSP] = msr_data;
1698 return X86EMUL_CONTINUE;
1702 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1704 struct decode_cache *c = &ctxt->decode;
1705 struct kvm_segment cs, ss;
1709 /* inject #GP if in real mode or Virtual 8086 mode */
1710 if (ctxt->mode == X86EMUL_MODE_REAL ||
1711 ctxt->mode == X86EMUL_MODE_VM86) {
1712 kvm_inject_gp(ctxt->vcpu, 0);
1713 return X86EMUL_UNHANDLEABLE;
1716 setup_syscalls_segments(ctxt, &cs, &ss);
1718 if ((c->rex_prefix & 0x8) != 0x0)
1719 usermode = X86EMUL_MODE_PROT64;
1721 usermode = X86EMUL_MODE_PROT32;
1725 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1727 case X86EMUL_MODE_PROT32:
1728 cs.selector = (u16)(msr_data + 16);
1729 if ((msr_data & 0xfffc) == 0x0) {
1730 kvm_inject_gp(ctxt->vcpu, 0);
1731 return X86EMUL_PROPAGATE_FAULT;
1733 ss.selector = (u16)(msr_data + 24);
1735 case X86EMUL_MODE_PROT64:
1736 cs.selector = (u16)(msr_data + 32);
1737 if (msr_data == 0x0) {
1738 kvm_inject_gp(ctxt->vcpu, 0);
1739 return X86EMUL_PROPAGATE_FAULT;
1741 ss.selector = cs.selector + 8;
1746 cs.selector |= SELECTOR_RPL_MASK;
1747 ss.selector |= SELECTOR_RPL_MASK;
1749 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1750 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1752 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1753 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1755 return X86EMUL_CONTINUE;
1758 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
1761 if (ctxt->mode == X86EMUL_MODE_REAL)
1763 if (ctxt->mode == X86EMUL_MODE_VM86)
1765 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1766 return kvm_x86_ops->get_cpl(ctxt->vcpu) > iopl;
1769 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1770 struct x86_emulate_ops *ops,
1773 struct kvm_segment tr_seg;
1776 u8 perm, bit_idx = port & 0x7;
1777 unsigned mask = (1 << len) - 1;
1779 kvm_get_segment(ctxt->vcpu, &tr_seg, VCPU_SREG_TR);
1780 if (tr_seg.unusable)
1782 if (tr_seg.limit < 103)
1784 r = ops->read_std(tr_seg.base + 102, &io_bitmap_ptr, 2, ctxt->vcpu,
1786 if (r != X86EMUL_CONTINUE)
1788 if (io_bitmap_ptr + port/8 > tr_seg.limit)
1790 r = ops->read_std(tr_seg.base + io_bitmap_ptr + port/8, &perm, 1,
1792 if (r != X86EMUL_CONTINUE)
1794 if ((perm >> bit_idx) & mask)
1799 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1800 struct x86_emulate_ops *ops,
1803 if (emulator_bad_iopl(ctxt))
1804 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1810 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1812 unsigned long memop = 0;
1814 unsigned long saved_eip = 0;
1815 struct decode_cache *c = &ctxt->decode;
1818 int rc = X86EMUL_CONTINUE;
1820 ctxt->interruptibility = 0;
1822 /* Shadow copy of register state. Committed on successful emulation.
1823 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1827 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1830 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
1831 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
1835 /* LOCK prefix is allowed only with some instructions */
1836 if (c->lock_prefix && !(c->d & Lock)) {
1837 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
1841 /* Privileged instruction can be executed only in CPL=0 */
1842 if ((c->d & Priv) && kvm_x86_ops->get_cpl(ctxt->vcpu)) {
1843 kvm_inject_gp(ctxt->vcpu, 0);
1847 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1848 memop = c->modrm_ea;
1850 if (c->rep_prefix && (c->d & String)) {
1851 /* All REP prefixes have the same first termination condition */
1852 if (c->regs[VCPU_REGS_RCX] == 0) {
1853 kvm_rip_write(ctxt->vcpu, c->eip);
1856 /* The second termination condition only applies for REPE
1857 * and REPNE. Test if the repeat string operation prefix is
1858 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1859 * corresponding termination condition according to:
1860 * - if REPE/REPZ and ZF = 0 then done
1861 * - if REPNE/REPNZ and ZF = 1 then done
1863 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1864 (c->b == 0xae) || (c->b == 0xaf)) {
1865 if ((c->rep_prefix == REPE_PREFIX) &&
1866 ((ctxt->eflags & EFLG_ZF) == 0)) {
1867 kvm_rip_write(ctxt->vcpu, c->eip);
1870 if ((c->rep_prefix == REPNE_PREFIX) &&
1871 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1872 kvm_rip_write(ctxt->vcpu, c->eip);
1876 c->regs[VCPU_REGS_RCX]--;
1877 c->eip = kvm_rip_read(ctxt->vcpu);
1880 if (c->src.type == OP_MEM) {
1881 c->src.ptr = (unsigned long *)memop;
1883 rc = ops->read_emulated((unsigned long)c->src.ptr,
1887 if (rc != X86EMUL_CONTINUE)
1889 c->src.orig_val = c->src.val;
1892 if (c->src2.type == OP_MEM) {
1893 c->src2.ptr = (unsigned long *)(memop + c->src.bytes);
1895 rc = ops->read_emulated((unsigned long)c->src2.ptr,
1899 if (rc != X86EMUL_CONTINUE)
1903 if ((c->d & DstMask) == ImplicitOps)
1907 if (c->dst.type == OP_MEM) {
1908 c->dst.ptr = (unsigned long *)memop;
1909 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1912 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1914 c->dst.ptr = (void *)c->dst.ptr +
1915 (c->src.val & mask) / 8;
1917 if (!(c->d & Mov)) {
1918 /* optimisation - avoid slow emulated read */
1919 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1923 if (rc != X86EMUL_CONTINUE)
1927 c->dst.orig_val = c->dst.val;
1937 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1939 case 0x06: /* push es */
1940 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1942 case 0x07: /* pop es */
1943 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1944 if (rc != X86EMUL_CONTINUE)
1949 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1951 case 0x0e: /* push cs */
1952 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1956 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1958 case 0x16: /* push ss */
1959 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1961 case 0x17: /* pop ss */
1962 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1963 if (rc != X86EMUL_CONTINUE)
1968 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1970 case 0x1e: /* push ds */
1971 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1973 case 0x1f: /* pop ds */
1974 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1975 if (rc != X86EMUL_CONTINUE)
1980 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1984 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1988 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1992 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1994 case 0x40 ... 0x47: /* inc r16/r32 */
1995 emulate_1op("inc", c->dst, ctxt->eflags);
1997 case 0x48 ... 0x4f: /* dec r16/r32 */
1998 emulate_1op("dec", c->dst, ctxt->eflags);
2000 case 0x50 ... 0x57: /* push reg */
2003 case 0x58 ... 0x5f: /* pop reg */
2005 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2006 if (rc != X86EMUL_CONTINUE)
2009 case 0x60: /* pusha */
2010 emulate_pusha(ctxt);
2012 case 0x61: /* popa */
2013 rc = emulate_popa(ctxt, ops);
2014 if (rc != X86EMUL_CONTINUE)
2017 case 0x63: /* movsxd */
2018 if (ctxt->mode != X86EMUL_MODE_PROT64)
2019 goto cannot_emulate;
2020 c->dst.val = (s32) c->src.val;
2022 case 0x68: /* push imm */
2023 case 0x6a: /* push imm8 */
2026 case 0x6c: /* insb */
2027 case 0x6d: /* insw/insd */
2028 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2029 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2030 kvm_inject_gp(ctxt->vcpu, 0);
2033 if (kvm_emulate_pio_string(ctxt->vcpu,
2035 (c->d & ByteOp) ? 1 : c->op_bytes,
2037 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2038 (ctxt->eflags & EFLG_DF),
2039 register_address(c, es_base(ctxt),
2040 c->regs[VCPU_REGS_RDI]),
2042 c->regs[VCPU_REGS_RDX]) == 0) {
2047 case 0x6e: /* outsb */
2048 case 0x6f: /* outsw/outsd */
2049 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2050 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2051 kvm_inject_gp(ctxt->vcpu, 0);
2054 if (kvm_emulate_pio_string(ctxt->vcpu,
2056 (c->d & ByteOp) ? 1 : c->op_bytes,
2058 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2059 (ctxt->eflags & EFLG_DF),
2061 seg_override_base(ctxt, c),
2062 c->regs[VCPU_REGS_RSI]),
2064 c->regs[VCPU_REGS_RDX]) == 0) {
2069 case 0x70 ... 0x7f: /* jcc (short) */
2070 if (test_cc(c->b, ctxt->eflags))
2071 jmp_rel(c, c->src.val);
2073 case 0x80 ... 0x83: /* Grp1 */
2074 switch (c->modrm_reg) {
2094 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2096 case 0x86 ... 0x87: /* xchg */
2098 /* Write back the register source. */
2099 switch (c->dst.bytes) {
2101 *(u8 *) c->src.ptr = (u8) c->dst.val;
2104 *(u16 *) c->src.ptr = (u16) c->dst.val;
2107 *c->src.ptr = (u32) c->dst.val;
2108 break; /* 64b reg: zero-extend */
2110 *c->src.ptr = c->dst.val;
2114 * Write back the memory destination with implicit LOCK
2117 c->dst.val = c->src.val;
2120 case 0x88 ... 0x8b: /* mov */
2122 case 0x8c: { /* mov r/m, sreg */
2123 struct kvm_segment segreg;
2125 if (c->modrm_reg <= 5)
2126 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
2128 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
2130 goto cannot_emulate;
2132 c->dst.val = segreg.selector;
2135 case 0x8d: /* lea r16/r32, m */
2136 c->dst.val = c->modrm_ea;
2138 case 0x8e: { /* mov seg, r/m16 */
2143 if (c->modrm_reg == VCPU_SREG_CS ||
2144 c->modrm_reg > VCPU_SREG_GS) {
2145 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
2149 if (c->modrm_reg == VCPU_SREG_SS)
2150 toggle_interruptibility(ctxt, KVM_X86_SHADOW_INT_MOV_SS);
2152 rc = kvm_load_segment_descriptor(ctxt->vcpu, sel, c->modrm_reg);
2154 c->dst.type = OP_NONE; /* Disable writeback. */
2157 case 0x8f: /* pop (sole member of Grp1a) */
2158 rc = emulate_grp1a(ctxt, ops);
2159 if (rc != X86EMUL_CONTINUE)
2162 case 0x90: /* nop / xchg r8,rax */
2163 if (!(c->rex_prefix & 1)) { /* nop */
2164 c->dst.type = OP_NONE;
2167 case 0x91 ... 0x97: /* xchg reg,rax */
2168 c->src.type = c->dst.type = OP_REG;
2169 c->src.bytes = c->dst.bytes = c->op_bytes;
2170 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2171 c->src.val = *(c->src.ptr);
2173 case 0x9c: /* pushf */
2174 c->src.val = (unsigned long) ctxt->eflags;
2177 case 0x9d: /* popf */
2178 c->dst.type = OP_REG;
2179 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2180 c->dst.bytes = c->op_bytes;
2181 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2182 if (rc != X86EMUL_CONTINUE)
2185 case 0xa0 ... 0xa1: /* mov */
2186 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2187 c->dst.val = c->src.val;
2189 case 0xa2 ... 0xa3: /* mov */
2190 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2192 case 0xa4 ... 0xa5: /* movs */
2193 c->dst.type = OP_MEM;
2194 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2195 c->dst.ptr = (unsigned long *)register_address(c,
2197 c->regs[VCPU_REGS_RDI]);
2198 rc = ops->read_emulated(register_address(c,
2199 seg_override_base(ctxt, c),
2200 c->regs[VCPU_REGS_RSI]),
2202 c->dst.bytes, ctxt->vcpu);
2203 if (rc != X86EMUL_CONTINUE)
2205 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2206 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2208 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2209 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2212 case 0xa6 ... 0xa7: /* cmps */
2213 c->src.type = OP_NONE; /* Disable writeback. */
2214 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2215 c->src.ptr = (unsigned long *)register_address(c,
2216 seg_override_base(ctxt, c),
2217 c->regs[VCPU_REGS_RSI]);
2218 rc = ops->read_emulated((unsigned long)c->src.ptr,
2222 if (rc != X86EMUL_CONTINUE)
2225 c->dst.type = OP_NONE; /* Disable writeback. */
2226 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2227 c->dst.ptr = (unsigned long *)register_address(c,
2229 c->regs[VCPU_REGS_RDI]);
2230 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2234 if (rc != X86EMUL_CONTINUE)
2237 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2239 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2241 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2242 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2244 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2245 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2249 case 0xaa ... 0xab: /* stos */
2250 c->dst.type = OP_MEM;
2251 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2252 c->dst.ptr = (unsigned long *)register_address(c,
2254 c->regs[VCPU_REGS_RDI]);
2255 c->dst.val = c->regs[VCPU_REGS_RAX];
2256 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2257 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2260 case 0xac ... 0xad: /* lods */
2261 c->dst.type = OP_REG;
2262 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2263 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2264 rc = ops->read_emulated(register_address(c,
2265 seg_override_base(ctxt, c),
2266 c->regs[VCPU_REGS_RSI]),
2270 if (rc != X86EMUL_CONTINUE)
2272 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2273 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2276 case 0xae ... 0xaf: /* scas */
2277 DPRINTF("Urk! I don't handle SCAS.\n");
2278 goto cannot_emulate;
2279 case 0xb0 ... 0xbf: /* mov r, imm */
2284 case 0xc3: /* ret */
2285 c->dst.type = OP_REG;
2286 c->dst.ptr = &c->eip;
2287 c->dst.bytes = c->op_bytes;
2288 goto pop_instruction;
2289 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2291 c->dst.val = c->src.val;
2293 case 0xcb: /* ret far */
2294 rc = emulate_ret_far(ctxt, ops);
2295 if (rc != X86EMUL_CONTINUE)
2298 case 0xd0 ... 0xd1: /* Grp2 */
2302 case 0xd2 ... 0xd3: /* Grp2 */
2303 c->src.val = c->regs[VCPU_REGS_RCX];
2306 case 0xe4: /* inb */
2311 case 0xe6: /* outb */
2312 case 0xe7: /* out */
2316 case 0xe8: /* call (near) */ {
2317 long int rel = c->src.val;
2318 c->src.val = (unsigned long) c->eip;
2323 case 0xe9: /* jmp rel */
2325 case 0xea: /* jmp far */
2327 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val,
2331 c->eip = c->src.val;
2334 jmp: /* jmp rel short */
2335 jmp_rel(c, c->src.val);
2336 c->dst.type = OP_NONE; /* Disable writeback. */
2338 case 0xec: /* in al,dx */
2339 case 0xed: /* in (e/r)ax,dx */
2340 port = c->regs[VCPU_REGS_RDX];
2343 case 0xee: /* out al,dx */
2344 case 0xef: /* out (e/r)ax,dx */
2345 port = c->regs[VCPU_REGS_RDX];
2348 if (!emulator_io_permited(ctxt, ops, port,
2349 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2350 kvm_inject_gp(ctxt->vcpu, 0);
2353 if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2354 (c->d & ByteOp) ? 1 : c->op_bytes,
2357 goto cannot_emulate;
2360 case 0xf4: /* hlt */
2361 ctxt->vcpu->arch.halt_request = 1;
2363 case 0xf5: /* cmc */
2364 /* complement carry flag from eflags reg */
2365 ctxt->eflags ^= EFLG_CF;
2366 c->dst.type = OP_NONE; /* Disable writeback. */
2368 case 0xf6 ... 0xf7: /* Grp3 */
2369 rc = emulate_grp3(ctxt, ops);
2370 if (rc != X86EMUL_CONTINUE)
2373 case 0xf8: /* clc */
2374 ctxt->eflags &= ~EFLG_CF;
2375 c->dst.type = OP_NONE; /* Disable writeback. */
2377 case 0xfa: /* cli */
2378 if (emulator_bad_iopl(ctxt))
2379 kvm_inject_gp(ctxt->vcpu, 0);
2381 ctxt->eflags &= ~X86_EFLAGS_IF;
2382 c->dst.type = OP_NONE; /* Disable writeback. */
2385 case 0xfb: /* sti */
2386 if (emulator_bad_iopl(ctxt))
2387 kvm_inject_gp(ctxt->vcpu, 0);
2389 toggle_interruptibility(ctxt, KVM_X86_SHADOW_INT_STI);
2390 ctxt->eflags |= X86_EFLAGS_IF;
2391 c->dst.type = OP_NONE; /* Disable writeback. */
2394 case 0xfc: /* cld */
2395 ctxt->eflags &= ~EFLG_DF;
2396 c->dst.type = OP_NONE; /* Disable writeback. */
2398 case 0xfd: /* std */
2399 ctxt->eflags |= EFLG_DF;
2400 c->dst.type = OP_NONE; /* Disable writeback. */
2402 case 0xfe: /* Grp4 */
2404 rc = emulate_grp45(ctxt, ops);
2405 if (rc != X86EMUL_CONTINUE)
2408 case 0xff: /* Grp5 */
2409 if (c->modrm_reg == 5)
2415 rc = writeback(ctxt, ops);
2416 if (rc != X86EMUL_CONTINUE)
2419 /* Commit shadow register state. */
2420 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2421 kvm_rip_write(ctxt->vcpu, c->eip);
2424 if (rc == X86EMUL_UNHANDLEABLE) {
2432 case 0x01: /* lgdt, lidt, lmsw */
2433 switch (c->modrm_reg) {
2435 unsigned long address;
2437 case 0: /* vmcall */
2438 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2439 goto cannot_emulate;
2441 rc = kvm_fix_hypercall(ctxt->vcpu);
2442 if (rc != X86EMUL_CONTINUE)
2445 /* Let the processor re-execute the fixed hypercall */
2446 c->eip = kvm_rip_read(ctxt->vcpu);
2447 /* Disable writeback. */
2448 c->dst.type = OP_NONE;
2451 rc = read_descriptor(ctxt, ops, c->src.ptr,
2452 &size, &address, c->op_bytes);
2453 if (rc != X86EMUL_CONTINUE)
2455 realmode_lgdt(ctxt->vcpu, size, address);
2456 /* Disable writeback. */
2457 c->dst.type = OP_NONE;
2459 case 3: /* lidt/vmmcall */
2460 if (c->modrm_mod == 3) {
2461 switch (c->modrm_rm) {
2463 rc = kvm_fix_hypercall(ctxt->vcpu);
2464 if (rc != X86EMUL_CONTINUE)
2468 goto cannot_emulate;
2471 rc = read_descriptor(ctxt, ops, c->src.ptr,
2474 if (rc != X86EMUL_CONTINUE)
2476 realmode_lidt(ctxt->vcpu, size, address);
2478 /* Disable writeback. */
2479 c->dst.type = OP_NONE;
2483 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2486 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2488 c->dst.type = OP_NONE;
2491 emulate_invlpg(ctxt->vcpu, memop);
2492 /* Disable writeback. */
2493 c->dst.type = OP_NONE;
2496 goto cannot_emulate;
2499 case 0x05: /* syscall */
2500 rc = emulate_syscall(ctxt);
2501 if (rc != X86EMUL_CONTINUE)
2507 emulate_clts(ctxt->vcpu);
2508 c->dst.type = OP_NONE;
2510 case 0x08: /* invd */
2511 case 0x09: /* wbinvd */
2512 case 0x0d: /* GrpP (prefetch) */
2513 case 0x18: /* Grp16 (prefetch/nop) */
2514 c->dst.type = OP_NONE;
2516 case 0x20: /* mov cr, reg */
2517 if (c->modrm_mod != 3)
2518 goto cannot_emulate;
2519 c->regs[c->modrm_rm] =
2520 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2521 c->dst.type = OP_NONE; /* no writeback */
2523 case 0x21: /* mov from dr to reg */
2524 if (c->modrm_mod != 3)
2525 goto cannot_emulate;
2526 if (emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]))
2527 goto cannot_emulate;
2528 rc = X86EMUL_CONTINUE;
2529 c->dst.type = OP_NONE; /* no writeback */
2531 case 0x22: /* mov reg, cr */
2532 if (c->modrm_mod != 3)
2533 goto cannot_emulate;
2534 realmode_set_cr(ctxt->vcpu,
2535 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2536 c->dst.type = OP_NONE;
2538 case 0x23: /* mov from reg to dr */
2539 if (c->modrm_mod != 3)
2540 goto cannot_emulate;
2541 if (emulator_set_dr(ctxt, c->modrm_reg, c->regs[c->modrm_rm]))
2542 goto cannot_emulate;
2543 rc = X86EMUL_CONTINUE;
2544 c->dst.type = OP_NONE; /* no writeback */
2548 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2549 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2550 if (kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
2551 kvm_inject_gp(ctxt->vcpu, 0);
2552 c->eip = kvm_rip_read(ctxt->vcpu);
2554 rc = X86EMUL_CONTINUE;
2555 c->dst.type = OP_NONE;
2559 if (kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
2560 kvm_inject_gp(ctxt->vcpu, 0);
2561 c->eip = kvm_rip_read(ctxt->vcpu);
2563 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2564 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2566 rc = X86EMUL_CONTINUE;
2567 c->dst.type = OP_NONE;
2569 case 0x34: /* sysenter */
2570 rc = emulate_sysenter(ctxt);
2571 if (rc != X86EMUL_CONTINUE)
2576 case 0x35: /* sysexit */
2577 rc = emulate_sysexit(ctxt);
2578 if (rc != X86EMUL_CONTINUE)
2583 case 0x40 ... 0x4f: /* cmov */
2584 c->dst.val = c->dst.orig_val = c->src.val;
2585 if (!test_cc(c->b, ctxt->eflags))
2586 c->dst.type = OP_NONE; /* no writeback */
2588 case 0x80 ... 0x8f: /* jnz rel, etc*/
2589 if (test_cc(c->b, ctxt->eflags))
2590 jmp_rel(c, c->src.val);
2591 c->dst.type = OP_NONE;
2593 case 0xa0: /* push fs */
2594 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2596 case 0xa1: /* pop fs */
2597 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2598 if (rc != X86EMUL_CONTINUE)
2603 c->dst.type = OP_NONE;
2604 /* only subword offset */
2605 c->src.val &= (c->dst.bytes << 3) - 1;
2606 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2608 case 0xa4: /* shld imm8, r, r/m */
2609 case 0xa5: /* shld cl, r, r/m */
2610 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2612 case 0xa8: /* push gs */
2613 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2615 case 0xa9: /* pop gs */
2616 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2617 if (rc != X86EMUL_CONTINUE)
2622 /* only subword offset */
2623 c->src.val &= (c->dst.bytes << 3) - 1;
2624 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2626 case 0xac: /* shrd imm8, r, r/m */
2627 case 0xad: /* shrd cl, r, r/m */
2628 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2630 case 0xae: /* clflush */
2632 case 0xb0 ... 0xb1: /* cmpxchg */
2634 * Save real source value, then compare EAX against
2637 c->src.orig_val = c->src.val;
2638 c->src.val = c->regs[VCPU_REGS_RAX];
2639 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2640 if (ctxt->eflags & EFLG_ZF) {
2641 /* Success: write back to memory. */
2642 c->dst.val = c->src.orig_val;
2644 /* Failure: write the value we saw to EAX. */
2645 c->dst.type = OP_REG;
2646 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2651 /* only subword offset */
2652 c->src.val &= (c->dst.bytes << 3) - 1;
2653 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2655 case 0xb6 ... 0xb7: /* movzx */
2656 c->dst.bytes = c->op_bytes;
2657 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2660 case 0xba: /* Grp8 */
2661 switch (c->modrm_reg & 3) {
2674 /* only subword offset */
2675 c->src.val &= (c->dst.bytes << 3) - 1;
2676 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2678 case 0xbe ... 0xbf: /* movsx */
2679 c->dst.bytes = c->op_bytes;
2680 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2683 case 0xc3: /* movnti */
2684 c->dst.bytes = c->op_bytes;
2685 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2688 case 0xc7: /* Grp9 (cmpxchg8b) */
2689 rc = emulate_grp9(ctxt, ops, memop);
2690 if (rc != X86EMUL_CONTINUE)
2692 c->dst.type = OP_NONE;
2698 DPRINTF("Cannot emulate %02x\n", c->b);