1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
80 /* Source 2 operand type */
81 #define Src2None (0<<29)
82 #define Src2CL (1<<29)
83 #define Src2ImmByte (2<<29)
84 #define Src2One (3<<29)
85 #define Src2Imm16 (4<<29)
86 #define Src2Mask (7<<29)
89 Group1_80, Group1_81, Group1_82, Group1_83,
90 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
94 static u32 opcode_table[256] = {
96 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
97 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
98 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
99 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
101 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
102 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
103 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
104 ImplicitOps | Stack | No64, 0,
106 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
107 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
108 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
109 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
111 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
112 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
113 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
114 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
116 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
117 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
118 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
120 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
121 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
124 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
125 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
128 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
129 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
133 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
135 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
137 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
138 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
140 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
141 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
143 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
144 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
147 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
148 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
149 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
151 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
152 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
155 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 Group | Group1_80, Group | Group1_81,
158 Group | Group1_82, Group | Group1_83,
159 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
160 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
162 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
163 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
164 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
165 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
167 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
169 0, 0, SrcImm | Src2Imm16 | No64, 0,
170 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
172 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
173 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
174 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
175 ByteOp | ImplicitOps | String, ImplicitOps | String,
177 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
178 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
179 ByteOp | ImplicitOps | String, ImplicitOps | String,
181 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
182 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
184 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
186 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
187 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
189 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
191 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
192 0, ImplicitOps | Stack, 0, 0,
193 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
195 0, 0, 0, ImplicitOps | Stack,
196 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
198 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
199 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
202 0, 0, 0, 0, 0, 0, 0, 0,
205 ByteOp | SrcImmUByte, SrcImmUByte,
206 ByteOp | SrcImmUByte, SrcImmUByte,
208 SrcImm | Stack, SrcImm | ImplicitOps,
209 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
210 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
211 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
214 ImplicitOps, ImplicitOps, Group | Group3_Byte, Group | Group3,
216 ImplicitOps, 0, ImplicitOps, ImplicitOps,
217 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
220 static u32 twobyte_table[256] = {
222 0, Group | GroupDual | Group7, 0, 0, 0, ImplicitOps, ImplicitOps, 0,
223 ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0,
225 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
227 ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0,
228 0, 0, 0, 0, 0, 0, 0, 0,
230 ImplicitOps, 0, ImplicitOps, 0,
231 ImplicitOps, ImplicitOps, 0, 0,
232 0, 0, 0, 0, 0, 0, 0, 0,
234 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
235 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
236 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
237 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
239 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
244 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
246 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
248 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
250 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
251 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
253 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
255 ImplicitOps | Stack, ImplicitOps | Stack,
256 0, DstMem | SrcReg | ModRM | BitOp,
257 DstMem | SrcReg | Src2ImmByte | ModRM,
258 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
260 ImplicitOps | Stack, ImplicitOps | Stack,
261 0, DstMem | SrcReg | ModRM | BitOp,
262 DstMem | SrcReg | Src2ImmByte | ModRM,
263 DstMem | SrcReg | Src2CL | ModRM,
266 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0,
267 DstMem | SrcReg | ModRM | BitOp,
268 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
269 DstReg | SrcMem16 | ModRM | Mov,
271 0, 0, Group | Group8, DstMem | SrcReg | ModRM | BitOp,
272 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
273 DstReg | SrcMem16 | ModRM | Mov,
275 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
276 0, 0, 0, Group | GroupDual | Group9,
277 0, 0, 0, 0, 0, 0, 0, 0,
279 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
281 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
283 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
286 static u32 group_table[] = {
288 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
289 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
290 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
291 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
293 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
294 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
295 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
296 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
298 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
299 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
300 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
301 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
303 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
304 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
305 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
306 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
308 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
310 ByteOp | SrcImm | DstMem | ModRM, 0,
311 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
314 DstMem | SrcImm | ModRM, 0,
315 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
318 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
321 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
322 SrcMem | ModRM | Stack, 0,
323 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
325 0, 0, ModRM | SrcMem, ModRM | SrcMem,
326 SrcNone | ModRM | DstMem | Mov, 0,
327 SrcMem16 | ModRM | Mov, SrcMem | ModRM | ByteOp,
330 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
331 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
333 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0,
336 static u32 group2_table[] = {
338 SrcNone | ModRM, 0, 0, SrcNone | ModRM,
339 SrcNone | ModRM | DstMem | Mov, 0,
340 SrcMem16 | ModRM | Mov, 0,
342 0, 0, 0, 0, 0, 0, 0, 0,
345 /* EFLAGS bit definitions. */
346 #define EFLG_ID (1<<21)
347 #define EFLG_VIP (1<<20)
348 #define EFLG_VIF (1<<19)
349 #define EFLG_AC (1<<18)
350 #define EFLG_VM (1<<17)
351 #define EFLG_RF (1<<16)
352 #define EFLG_IOPL (3<<12)
353 #define EFLG_NT (1<<14)
354 #define EFLG_OF (1<<11)
355 #define EFLG_DF (1<<10)
356 #define EFLG_IF (1<<9)
357 #define EFLG_TF (1<<8)
358 #define EFLG_SF (1<<7)
359 #define EFLG_ZF (1<<6)
360 #define EFLG_AF (1<<4)
361 #define EFLG_PF (1<<2)
362 #define EFLG_CF (1<<0)
365 * Instruction emulation:
366 * Most instructions are emulated directly via a fragment of inline assembly
367 * code. This allows us to save/restore EFLAGS and thus very easily pick up
368 * any modified flags.
371 #if defined(CONFIG_X86_64)
372 #define _LO32 "k" /* force 32-bit operand */
373 #define _STK "%%rsp" /* stack pointer */
374 #elif defined(__i386__)
375 #define _LO32 "" /* force 32-bit operand */
376 #define _STK "%%esp" /* stack pointer */
380 * These EFLAGS bits are restored from saved value during emulation, and
381 * any changes are written back to the saved value after emulation.
383 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
385 /* Before executing instruction: restore necessary bits in EFLAGS. */
386 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
387 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
388 "movl %"_sav",%"_LO32 _tmp"; " \
391 "movl %"_msk",%"_LO32 _tmp"; " \
392 "andl %"_LO32 _tmp",("_STK"); " \
394 "notl %"_LO32 _tmp"; " \
395 "andl %"_LO32 _tmp",("_STK"); " \
396 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
398 "orl %"_LO32 _tmp",("_STK"); " \
402 /* After executing instruction: write-back necessary bits in EFLAGS. */
403 #define _POST_EFLAGS(_sav, _msk, _tmp) \
404 /* _sav |= EFLAGS & _msk; */ \
407 "andl %"_msk",%"_LO32 _tmp"; " \
408 "orl %"_LO32 _tmp",%"_sav"; "
416 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
418 __asm__ __volatile__ ( \
419 _PRE_EFLAGS("0", "4", "2") \
420 _op _suffix " %"_x"3,%1; " \
421 _POST_EFLAGS("0", "4", "2") \
422 : "=m" (_eflags), "=m" ((_dst).val), \
424 : _y ((_src).val), "i" (EFLAGS_MASK)); \
428 /* Raw emulation: instruction has two explicit operands. */
429 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
431 unsigned long _tmp; \
433 switch ((_dst).bytes) { \
435 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
438 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
441 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
446 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
448 unsigned long _tmp; \
449 switch ((_dst).bytes) { \
451 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
454 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
455 _wx, _wy, _lx, _ly, _qx, _qy); \
460 /* Source operand is byte-sized and may be restricted to just %cl. */
461 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
462 __emulate_2op(_op, _src, _dst, _eflags, \
463 "b", "c", "b", "c", "b", "c", "b", "c")
465 /* Source operand is byte, word, long or quad sized. */
466 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
467 __emulate_2op(_op, _src, _dst, _eflags, \
468 "b", "q", "w", "r", _LO32, "r", "", "r")
470 /* Source operand is word, long or quad sized. */
471 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
472 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
473 "w", "r", _LO32, "r", "", "r")
475 /* Instruction has three operands and one operand is stored in ECX register */
476 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
478 unsigned long _tmp; \
479 _type _clv = (_cl).val; \
480 _type _srcv = (_src).val; \
481 _type _dstv = (_dst).val; \
483 __asm__ __volatile__ ( \
484 _PRE_EFLAGS("0", "5", "2") \
485 _op _suffix " %4,%1 \n" \
486 _POST_EFLAGS("0", "5", "2") \
487 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
488 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
491 (_cl).val = (unsigned long) _clv; \
492 (_src).val = (unsigned long) _srcv; \
493 (_dst).val = (unsigned long) _dstv; \
496 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
498 switch ((_dst).bytes) { \
500 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
501 "w", unsigned short); \
504 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
505 "l", unsigned int); \
508 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
509 "q", unsigned long)); \
514 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
516 unsigned long _tmp; \
518 __asm__ __volatile__ ( \
519 _PRE_EFLAGS("0", "3", "2") \
520 _op _suffix " %1; " \
521 _POST_EFLAGS("0", "3", "2") \
522 : "=m" (_eflags), "+m" ((_dst).val), \
524 : "i" (EFLAGS_MASK)); \
527 /* Instruction has only one explicit operand (no source operand). */
528 #define emulate_1op(_op, _dst, _eflags) \
530 switch ((_dst).bytes) { \
531 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
532 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
533 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
534 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
538 /* Fetch next part of the instruction being emulated. */
539 #define insn_fetch(_type, _size, _eip) \
540 ({ unsigned long _x; \
541 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
548 static inline unsigned long ad_mask(struct decode_cache *c)
550 return (1UL << (c->ad_bytes << 3)) - 1;
553 /* Access/update address held in a register, based on addressing mode. */
554 static inline unsigned long
555 address_mask(struct decode_cache *c, unsigned long reg)
557 if (c->ad_bytes == sizeof(unsigned long))
560 return reg & ad_mask(c);
563 static inline unsigned long
564 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
566 return base + address_mask(c, reg);
570 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
572 if (c->ad_bytes == sizeof(unsigned long))
575 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
578 static inline void jmp_rel(struct decode_cache *c, int rel)
580 register_address_increment(c, &c->eip, rel);
583 static void set_seg_override(struct decode_cache *c, int seg)
585 c->has_seg_override = true;
586 c->seg_override = seg;
589 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
591 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
594 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
597 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
598 struct decode_cache *c)
600 if (!c->has_seg_override)
603 return seg_base(ctxt, c->seg_override);
606 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
608 return seg_base(ctxt, VCPU_SREG_ES);
611 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
613 return seg_base(ctxt, VCPU_SREG_SS);
616 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
617 struct x86_emulate_ops *ops,
618 unsigned long linear, u8 *dest)
620 struct fetch_cache *fc = &ctxt->decode.fetch;
624 if (linear < fc->start || linear >= fc->end) {
625 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
626 rc = ops->fetch(linear, fc->data, size, ctxt->vcpu, NULL);
630 fc->end = linear + size;
632 *dest = fc->data[linear - fc->start];
636 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
637 struct x86_emulate_ops *ops,
638 unsigned long eip, void *dest, unsigned size)
642 /* x86 instructions are limited to 15 bytes. */
643 if (eip + size - ctxt->decode.eip_orig > 15)
644 return X86EMUL_UNHANDLEABLE;
645 eip += ctxt->cs_base;
647 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
655 * Given the 'reg' portion of a ModRM byte, and a register block, return a
656 * pointer into the block that addresses the relevant register.
657 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
659 static void *decode_register(u8 modrm_reg, unsigned long *regs,
664 p = ®s[modrm_reg];
665 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
666 p = (unsigned char *)®s[modrm_reg & 3] + 1;
670 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
671 struct x86_emulate_ops *ops,
673 u16 *size, unsigned long *address, int op_bytes)
680 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
684 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
689 static int test_cc(unsigned int condition, unsigned int flags)
693 switch ((condition & 15) >> 1) {
695 rc |= (flags & EFLG_OF);
697 case 1: /* b/c/nae */
698 rc |= (flags & EFLG_CF);
701 rc |= (flags & EFLG_ZF);
704 rc |= (flags & (EFLG_CF|EFLG_ZF));
707 rc |= (flags & EFLG_SF);
710 rc |= (flags & EFLG_PF);
713 rc |= (flags & EFLG_ZF);
716 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
720 /* Odd condition identifiers (lsb == 1) have inverted sense. */
721 return (!!rc ^ (condition & 1));
724 static void decode_register_operand(struct operand *op,
725 struct decode_cache *c,
728 unsigned reg = c->modrm_reg;
729 int highbyte_regs = c->rex_prefix == 0;
732 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
734 if ((c->d & ByteOp) && !inhibit_bytereg) {
735 op->ptr = decode_register(reg, c->regs, highbyte_regs);
736 op->val = *(u8 *)op->ptr;
739 op->ptr = decode_register(reg, c->regs, 0);
740 op->bytes = c->op_bytes;
743 op->val = *(u16 *)op->ptr;
746 op->val = *(u32 *)op->ptr;
749 op->val = *(u64 *) op->ptr;
753 op->orig_val = op->val;
756 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
757 struct x86_emulate_ops *ops)
759 struct decode_cache *c = &ctxt->decode;
761 int index_reg = 0, base_reg = 0, scale;
765 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
766 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
767 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
770 c->modrm = insn_fetch(u8, 1, c->eip);
771 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
772 c->modrm_reg |= (c->modrm & 0x38) >> 3;
773 c->modrm_rm |= (c->modrm & 0x07);
777 if (c->modrm_mod == 3) {
778 c->modrm_ptr = decode_register(c->modrm_rm,
779 c->regs, c->d & ByteOp);
780 c->modrm_val = *(unsigned long *)c->modrm_ptr;
784 if (c->ad_bytes == 2) {
785 unsigned bx = c->regs[VCPU_REGS_RBX];
786 unsigned bp = c->regs[VCPU_REGS_RBP];
787 unsigned si = c->regs[VCPU_REGS_RSI];
788 unsigned di = c->regs[VCPU_REGS_RDI];
790 /* 16-bit ModR/M decode. */
791 switch (c->modrm_mod) {
793 if (c->modrm_rm == 6)
794 c->modrm_ea += insn_fetch(u16, 2, c->eip);
797 c->modrm_ea += insn_fetch(s8, 1, c->eip);
800 c->modrm_ea += insn_fetch(u16, 2, c->eip);
803 switch (c->modrm_rm) {
805 c->modrm_ea += bx + si;
808 c->modrm_ea += bx + di;
811 c->modrm_ea += bp + si;
814 c->modrm_ea += bp + di;
823 if (c->modrm_mod != 0)
830 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
831 (c->modrm_rm == 6 && c->modrm_mod != 0))
832 if (!c->has_seg_override)
833 set_seg_override(c, VCPU_SREG_SS);
834 c->modrm_ea = (u16)c->modrm_ea;
836 /* 32/64-bit ModR/M decode. */
837 if ((c->modrm_rm & 7) == 4) {
838 sib = insn_fetch(u8, 1, c->eip);
839 index_reg |= (sib >> 3) & 7;
843 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
844 c->modrm_ea += insn_fetch(s32, 4, c->eip);
846 c->modrm_ea += c->regs[base_reg];
848 c->modrm_ea += c->regs[index_reg] << scale;
849 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
850 if (ctxt->mode == X86EMUL_MODE_PROT64)
853 c->modrm_ea += c->regs[c->modrm_rm];
854 switch (c->modrm_mod) {
856 if (c->modrm_rm == 5)
857 c->modrm_ea += insn_fetch(s32, 4, c->eip);
860 c->modrm_ea += insn_fetch(s8, 1, c->eip);
863 c->modrm_ea += insn_fetch(s32, 4, c->eip);
871 static int decode_abs(struct x86_emulate_ctxt *ctxt,
872 struct x86_emulate_ops *ops)
874 struct decode_cache *c = &ctxt->decode;
877 switch (c->ad_bytes) {
879 c->modrm_ea = insn_fetch(u16, 2, c->eip);
882 c->modrm_ea = insn_fetch(u32, 4, c->eip);
885 c->modrm_ea = insn_fetch(u64, 8, c->eip);
893 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
895 struct decode_cache *c = &ctxt->decode;
897 int mode = ctxt->mode;
898 int def_op_bytes, def_ad_bytes, group;
900 /* Shadow copy of register state. Committed on successful emulation. */
902 memset(c, 0, sizeof(struct decode_cache));
903 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
904 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
905 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
908 case X86EMUL_MODE_REAL:
909 case X86EMUL_MODE_VM86:
910 case X86EMUL_MODE_PROT16:
911 def_op_bytes = def_ad_bytes = 2;
913 case X86EMUL_MODE_PROT32:
914 def_op_bytes = def_ad_bytes = 4;
917 case X86EMUL_MODE_PROT64:
926 c->op_bytes = def_op_bytes;
927 c->ad_bytes = def_ad_bytes;
929 /* Legacy prefixes. */
931 switch (c->b = insn_fetch(u8, 1, c->eip)) {
932 case 0x66: /* operand-size override */
933 /* switch between 2/4 bytes */
934 c->op_bytes = def_op_bytes ^ 6;
936 case 0x67: /* address-size override */
937 if (mode == X86EMUL_MODE_PROT64)
938 /* switch between 4/8 bytes */
939 c->ad_bytes = def_ad_bytes ^ 12;
941 /* switch between 2/4 bytes */
942 c->ad_bytes = def_ad_bytes ^ 6;
944 case 0x26: /* ES override */
945 case 0x2e: /* CS override */
946 case 0x36: /* SS override */
947 case 0x3e: /* DS override */
948 set_seg_override(c, (c->b >> 3) & 3);
950 case 0x64: /* FS override */
951 case 0x65: /* GS override */
952 set_seg_override(c, c->b & 7);
954 case 0x40 ... 0x4f: /* REX */
955 if (mode != X86EMUL_MODE_PROT64)
957 c->rex_prefix = c->b;
959 case 0xf0: /* LOCK */
962 case 0xf2: /* REPNE/REPNZ */
963 c->rep_prefix = REPNE_PREFIX;
965 case 0xf3: /* REP/REPE/REPZ */
966 c->rep_prefix = REPE_PREFIX;
972 /* Any legacy prefix after a REX prefix nullifies its effect. */
981 if (c->rex_prefix & 8)
982 c->op_bytes = 8; /* REX.W */
984 /* Opcode byte(s). */
985 c->d = opcode_table[c->b];
987 /* Two-byte opcode? */
990 c->b = insn_fetch(u8, 1, c->eip);
991 c->d = twobyte_table[c->b];
995 if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
996 kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");;
1001 group = c->d & GroupMask;
1002 c->modrm = insn_fetch(u8, 1, c->eip);
1005 group = (group << 3) + ((c->modrm >> 3) & 7);
1006 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1007 c->d = group2_table[group];
1009 c->d = group_table[group];
1014 DPRINTF("Cannot emulate %02x\n", c->b);
1018 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1021 /* ModRM and SIB bytes. */
1023 rc = decode_modrm(ctxt, ops);
1024 else if (c->d & MemAbs)
1025 rc = decode_abs(ctxt, ops);
1029 if (!c->has_seg_override)
1030 set_seg_override(c, VCPU_SREG_DS);
1032 if (!(!c->twobyte && c->b == 0x8d))
1033 c->modrm_ea += seg_override_base(ctxt, c);
1035 if (c->ad_bytes != 8)
1036 c->modrm_ea = (u32)c->modrm_ea;
1038 * Decode and fetch the source operand: register, memory
1041 switch (c->d & SrcMask) {
1045 decode_register_operand(&c->src, c, 0);
1054 c->src.bytes = (c->d & ByteOp) ? 1 :
1056 /* Don't fetch the address for invlpg: it could be unmapped. */
1057 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1061 * For instructions with a ModR/M byte, switch to register
1062 * access if Mod = 3.
1064 if ((c->d & ModRM) && c->modrm_mod == 3) {
1065 c->src.type = OP_REG;
1066 c->src.val = c->modrm_val;
1067 c->src.ptr = c->modrm_ptr;
1070 c->src.type = OP_MEM;
1074 c->src.type = OP_IMM;
1075 c->src.ptr = (unsigned long *)c->eip;
1076 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1077 if (c->src.bytes == 8)
1079 /* NB. Immediates are sign-extended as necessary. */
1080 switch (c->src.bytes) {
1082 c->src.val = insn_fetch(s8, 1, c->eip);
1085 c->src.val = insn_fetch(s16, 2, c->eip);
1088 c->src.val = insn_fetch(s32, 4, c->eip);
1091 if ((c->d & SrcMask) == SrcImmU) {
1092 switch (c->src.bytes) {
1097 c->src.val &= 0xffff;
1100 c->src.val &= 0xffffffff;
1107 c->src.type = OP_IMM;
1108 c->src.ptr = (unsigned long *)c->eip;
1110 if ((c->d & SrcMask) == SrcImmByte)
1111 c->src.val = insn_fetch(s8, 1, c->eip);
1113 c->src.val = insn_fetch(u8, 1, c->eip);
1122 * Decode and fetch the second source operand: register, memory
1125 switch (c->d & Src2Mask) {
1130 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1133 c->src2.type = OP_IMM;
1134 c->src2.ptr = (unsigned long *)c->eip;
1136 c->src2.val = insn_fetch(u8, 1, c->eip);
1139 c->src2.type = OP_IMM;
1140 c->src2.ptr = (unsigned long *)c->eip;
1142 c->src2.val = insn_fetch(u16, 2, c->eip);
1150 /* Decode and fetch the destination operand: register or memory. */
1151 switch (c->d & DstMask) {
1153 /* Special instructions do their own operand decoding. */
1156 decode_register_operand(&c->dst, c,
1157 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1160 if ((c->d & ModRM) && c->modrm_mod == 3) {
1161 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1162 c->dst.type = OP_REG;
1163 c->dst.val = c->dst.orig_val = c->modrm_val;
1164 c->dst.ptr = c->modrm_ptr;
1167 c->dst.type = OP_MEM;
1170 c->dst.type = OP_REG;
1171 c->dst.bytes = c->op_bytes;
1172 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1173 switch (c->op_bytes) {
1175 c->dst.val = *(u8 *)c->dst.ptr;
1178 c->dst.val = *(u16 *)c->dst.ptr;
1181 c->dst.val = *(u32 *)c->dst.ptr;
1184 c->dst.orig_val = c->dst.val;
1188 if (c->rip_relative)
1189 c->modrm_ea += c->eip;
1192 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1195 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1197 struct decode_cache *c = &ctxt->decode;
1199 c->dst.type = OP_MEM;
1200 c->dst.bytes = c->op_bytes;
1201 c->dst.val = c->src.val;
1202 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1203 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1204 c->regs[VCPU_REGS_RSP]);
1207 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1208 struct x86_emulate_ops *ops,
1209 void *dest, int len)
1211 struct decode_cache *c = &ctxt->decode;
1214 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1215 c->regs[VCPU_REGS_RSP]),
1216 dest, len, ctxt->vcpu);
1217 if (rc != X86EMUL_CONTINUE)
1220 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1224 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1225 struct x86_emulate_ops *ops,
1226 void *dest, int len)
1229 unsigned long val, change_mask;
1230 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1231 int cpl = kvm_x86_ops->get_cpl(ctxt->vcpu);
1233 rc = emulate_pop(ctxt, ops, &val, len);
1234 if (rc != X86EMUL_CONTINUE)
1237 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1238 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1240 switch(ctxt->mode) {
1241 case X86EMUL_MODE_PROT64:
1242 case X86EMUL_MODE_PROT32:
1243 case X86EMUL_MODE_PROT16:
1245 change_mask |= EFLG_IOPL;
1247 change_mask |= EFLG_IF;
1249 case X86EMUL_MODE_VM86:
1251 kvm_inject_gp(ctxt->vcpu, 0);
1252 return X86EMUL_PROPAGATE_FAULT;
1254 change_mask |= EFLG_IF;
1256 default: /* real mode */
1257 change_mask |= (EFLG_IOPL | EFLG_IF);
1261 *(unsigned long *)dest =
1262 (ctxt->eflags & ~change_mask) | (val & change_mask);
1267 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1269 struct decode_cache *c = &ctxt->decode;
1270 struct kvm_segment segment;
1272 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1274 c->src.val = segment.selector;
1278 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1279 struct x86_emulate_ops *ops, int seg)
1281 struct decode_cache *c = &ctxt->decode;
1282 unsigned long selector;
1285 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1289 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, 1, seg);
1293 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1295 struct decode_cache *c = &ctxt->decode;
1296 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1297 int reg = VCPU_REGS_RAX;
1299 while (reg <= VCPU_REGS_RDI) {
1300 (reg == VCPU_REGS_RSP) ?
1301 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1308 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1309 struct x86_emulate_ops *ops)
1311 struct decode_cache *c = &ctxt->decode;
1313 int reg = VCPU_REGS_RDI;
1315 while (reg >= VCPU_REGS_RAX) {
1316 if (reg == VCPU_REGS_RSP) {
1317 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1322 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1330 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1331 struct x86_emulate_ops *ops)
1333 struct decode_cache *c = &ctxt->decode;
1336 rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1342 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1344 struct decode_cache *c = &ctxt->decode;
1345 switch (c->modrm_reg) {
1347 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1350 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1353 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1356 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1358 case 4: /* sal/shl */
1359 case 6: /* sal/shl */
1360 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1363 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1366 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1371 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1372 struct x86_emulate_ops *ops)
1374 struct decode_cache *c = &ctxt->decode;
1377 switch (c->modrm_reg) {
1378 case 0 ... 1: /* test */
1379 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1382 c->dst.val = ~c->dst.val;
1385 emulate_1op("neg", c->dst, ctxt->eflags);
1388 DPRINTF("Cannot emulate %02x\n", c->b);
1389 rc = X86EMUL_UNHANDLEABLE;
1395 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1396 struct x86_emulate_ops *ops)
1398 struct decode_cache *c = &ctxt->decode;
1400 switch (c->modrm_reg) {
1402 emulate_1op("inc", c->dst, ctxt->eflags);
1405 emulate_1op("dec", c->dst, ctxt->eflags);
1407 case 2: /* call near abs */ {
1410 c->eip = c->src.val;
1411 c->src.val = old_eip;
1415 case 4: /* jmp abs */
1416 c->eip = c->src.val;
1425 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1426 struct x86_emulate_ops *ops,
1427 unsigned long memop)
1429 struct decode_cache *c = &ctxt->decode;
1433 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1434 if (rc != X86EMUL_CONTINUE)
1437 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1438 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1440 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1441 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1442 ctxt->eflags &= ~EFLG_ZF;
1445 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1446 (u32) c->regs[VCPU_REGS_RBX];
1448 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1449 if (rc != X86EMUL_CONTINUE)
1451 ctxt->eflags |= EFLG_ZF;
1456 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1457 struct x86_emulate_ops *ops)
1459 struct decode_cache *c = &ctxt->decode;
1463 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1466 if (c->op_bytes == 4)
1467 c->eip = (u32)c->eip;
1468 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1471 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, 1, VCPU_SREG_CS);
1475 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1476 struct x86_emulate_ops *ops)
1479 struct decode_cache *c = &ctxt->decode;
1481 switch (c->dst.type) {
1483 /* The 4-byte case *is* correct:
1484 * in 64-bit mode we zero-extend.
1486 switch (c->dst.bytes) {
1488 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1491 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1494 *c->dst.ptr = (u32)c->dst.val;
1495 break; /* 64b: zero-ext */
1497 *c->dst.ptr = c->dst.val;
1503 rc = ops->cmpxchg_emulated(
1504 (unsigned long)c->dst.ptr,
1510 rc = ops->write_emulated(
1511 (unsigned long)c->dst.ptr,
1515 if (rc != X86EMUL_CONTINUE)
1527 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1529 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1531 * an sti; sti; sequence only disable interrupts for the first
1532 * instruction. So, if the last instruction, be it emulated or
1533 * not, left the system with the INT_STI flag enabled, it
1534 * means that the last instruction is an sti. We should not
1535 * leave the flag on in this case. The same goes for mov ss
1537 if (!(int_shadow & mask))
1538 ctxt->interruptibility = mask;
1542 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1543 struct kvm_segment *cs, struct kvm_segment *ss)
1545 memset(cs, 0, sizeof(struct kvm_segment));
1546 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1547 memset(ss, 0, sizeof(struct kvm_segment));
1549 cs->l = 0; /* will be adjusted later */
1550 cs->base = 0; /* flat segment */
1551 cs->g = 1; /* 4kb granularity */
1552 cs->limit = 0xffffffff; /* 4GB limit */
1553 cs->type = 0x0b; /* Read, Execute, Accessed */
1555 cs->dpl = 0; /* will be adjusted later */
1560 ss->base = 0; /* flat segment */
1561 ss->limit = 0xffffffff; /* 4GB limit */
1562 ss->g = 1; /* 4kb granularity */
1564 ss->type = 0x03; /* Read/Write, Accessed */
1565 ss->db = 1; /* 32bit stack segment */
1571 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1573 struct decode_cache *c = &ctxt->decode;
1574 struct kvm_segment cs, ss;
1577 /* syscall is not available in real mode */
1578 if (c->lock_prefix || ctxt->mode == X86EMUL_MODE_REAL
1579 || ctxt->mode == X86EMUL_MODE_VM86)
1582 setup_syscalls_segments(ctxt, &cs, &ss);
1584 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1586 cs.selector = (u16)(msr_data & 0xfffc);
1587 ss.selector = (u16)(msr_data + 8);
1589 if (is_long_mode(ctxt->vcpu)) {
1593 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1594 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1596 c->regs[VCPU_REGS_RCX] = c->eip;
1597 if (is_long_mode(ctxt->vcpu)) {
1598 #ifdef CONFIG_X86_64
1599 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1601 kvm_x86_ops->get_msr(ctxt->vcpu,
1602 ctxt->mode == X86EMUL_MODE_PROT64 ?
1603 MSR_LSTAR : MSR_CSTAR, &msr_data);
1606 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1607 ctxt->eflags &= ~(msr_data | EFLG_RF);
1611 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1612 c->eip = (u32)msr_data;
1614 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1621 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1623 struct decode_cache *c = &ctxt->decode;
1624 struct kvm_segment cs, ss;
1627 /* inject #UD if LOCK prefix is used */
1631 /* inject #GP if in real mode */
1632 if (ctxt->mode == X86EMUL_MODE_REAL) {
1633 kvm_inject_gp(ctxt->vcpu, 0);
1637 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1638 * Therefore, we inject an #UD.
1640 if (ctxt->mode == X86EMUL_MODE_PROT64)
1643 setup_syscalls_segments(ctxt, &cs, &ss);
1645 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1646 switch (ctxt->mode) {
1647 case X86EMUL_MODE_PROT32:
1648 if ((msr_data & 0xfffc) == 0x0) {
1649 kvm_inject_gp(ctxt->vcpu, 0);
1653 case X86EMUL_MODE_PROT64:
1654 if (msr_data == 0x0) {
1655 kvm_inject_gp(ctxt->vcpu, 0);
1661 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1662 cs.selector = (u16)msr_data;
1663 cs.selector &= ~SELECTOR_RPL_MASK;
1664 ss.selector = cs.selector + 8;
1665 ss.selector &= ~SELECTOR_RPL_MASK;
1666 if (ctxt->mode == X86EMUL_MODE_PROT64
1667 || is_long_mode(ctxt->vcpu)) {
1672 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1673 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1675 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1678 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1679 c->regs[VCPU_REGS_RSP] = msr_data;
1685 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1687 struct decode_cache *c = &ctxt->decode;
1688 struct kvm_segment cs, ss;
1692 /* inject #UD if LOCK prefix is used */
1696 /* inject #GP if in real mode or Virtual 8086 mode */
1697 if (ctxt->mode == X86EMUL_MODE_REAL ||
1698 ctxt->mode == X86EMUL_MODE_VM86) {
1699 kvm_inject_gp(ctxt->vcpu, 0);
1703 /* sysexit must be called from CPL 0 */
1704 if (kvm_x86_ops->get_cpl(ctxt->vcpu) != 0) {
1705 kvm_inject_gp(ctxt->vcpu, 0);
1709 setup_syscalls_segments(ctxt, &cs, &ss);
1711 if ((c->rex_prefix & 0x8) != 0x0)
1712 usermode = X86EMUL_MODE_PROT64;
1714 usermode = X86EMUL_MODE_PROT32;
1718 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1720 case X86EMUL_MODE_PROT32:
1721 cs.selector = (u16)(msr_data + 16);
1722 if ((msr_data & 0xfffc) == 0x0) {
1723 kvm_inject_gp(ctxt->vcpu, 0);
1726 ss.selector = (u16)(msr_data + 24);
1728 case X86EMUL_MODE_PROT64:
1729 cs.selector = (u16)(msr_data + 32);
1730 if (msr_data == 0x0) {
1731 kvm_inject_gp(ctxt->vcpu, 0);
1734 ss.selector = cs.selector + 8;
1739 cs.selector |= SELECTOR_RPL_MASK;
1740 ss.selector |= SELECTOR_RPL_MASK;
1742 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1743 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1745 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1746 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1751 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
1754 if (ctxt->mode == X86EMUL_MODE_REAL)
1756 if (ctxt->mode == X86EMUL_MODE_VM86)
1758 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1759 return kvm_x86_ops->get_cpl(ctxt->vcpu) > iopl;
1762 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1763 struct x86_emulate_ops *ops,
1766 struct kvm_segment tr_seg;
1769 u8 perm, bit_idx = port & 0x7;
1770 unsigned mask = (1 << len) - 1;
1772 kvm_get_segment(ctxt->vcpu, &tr_seg, VCPU_SREG_TR);
1773 if (tr_seg.unusable)
1775 if (tr_seg.limit < 103)
1777 r = ops->read_std(tr_seg.base + 102, &io_bitmap_ptr, 2, ctxt->vcpu,
1779 if (r != X86EMUL_CONTINUE)
1781 if (io_bitmap_ptr + port/8 > tr_seg.limit)
1783 r = ops->read_std(tr_seg.base + io_bitmap_ptr + port/8, &perm, 1,
1785 if (r != X86EMUL_CONTINUE)
1787 if ((perm >> bit_idx) & mask)
1792 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1793 struct x86_emulate_ops *ops,
1796 if (emulator_bad_iopl(ctxt))
1797 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1803 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1805 unsigned long memop = 0;
1807 unsigned long saved_eip = 0;
1808 struct decode_cache *c = &ctxt->decode;
1813 ctxt->interruptibility = 0;
1815 /* Shadow copy of register state. Committed on successful emulation.
1816 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1820 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1823 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1824 memop = c->modrm_ea;
1826 if (c->rep_prefix && (c->d & String)) {
1827 /* All REP prefixes have the same first termination condition */
1828 if (c->regs[VCPU_REGS_RCX] == 0) {
1829 kvm_rip_write(ctxt->vcpu, c->eip);
1832 /* The second termination condition only applies for REPE
1833 * and REPNE. Test if the repeat string operation prefix is
1834 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1835 * corresponding termination condition according to:
1836 * - if REPE/REPZ and ZF = 0 then done
1837 * - if REPNE/REPNZ and ZF = 1 then done
1839 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1840 (c->b == 0xae) || (c->b == 0xaf)) {
1841 if ((c->rep_prefix == REPE_PREFIX) &&
1842 ((ctxt->eflags & EFLG_ZF) == 0)) {
1843 kvm_rip_write(ctxt->vcpu, c->eip);
1846 if ((c->rep_prefix == REPNE_PREFIX) &&
1847 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1848 kvm_rip_write(ctxt->vcpu, c->eip);
1852 c->regs[VCPU_REGS_RCX]--;
1853 c->eip = kvm_rip_read(ctxt->vcpu);
1856 if (c->src.type == OP_MEM) {
1857 c->src.ptr = (unsigned long *)memop;
1859 rc = ops->read_emulated((unsigned long)c->src.ptr,
1863 if (rc != X86EMUL_CONTINUE)
1865 c->src.orig_val = c->src.val;
1868 if ((c->d & DstMask) == ImplicitOps)
1872 if (c->dst.type == OP_MEM) {
1873 c->dst.ptr = (unsigned long *)memop;
1874 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1877 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1879 c->dst.ptr = (void *)c->dst.ptr +
1880 (c->src.val & mask) / 8;
1882 if (!(c->d & Mov)) {
1883 /* optimisation - avoid slow emulated read */
1884 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1888 if (rc != X86EMUL_CONTINUE)
1892 c->dst.orig_val = c->dst.val;
1902 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1904 case 0x06: /* push es */
1905 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1907 case 0x07: /* pop es */
1908 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1914 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1916 case 0x0e: /* push cs */
1917 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1921 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1923 case 0x16: /* push ss */
1924 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1926 case 0x17: /* pop ss */
1927 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1933 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1935 case 0x1e: /* push ds */
1936 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1938 case 0x1f: /* pop ds */
1939 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1945 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1949 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1953 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1957 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1959 case 0x40 ... 0x47: /* inc r16/r32 */
1960 emulate_1op("inc", c->dst, ctxt->eflags);
1962 case 0x48 ... 0x4f: /* dec r16/r32 */
1963 emulate_1op("dec", c->dst, ctxt->eflags);
1965 case 0x50 ... 0x57: /* push reg */
1968 case 0x58 ... 0x5f: /* pop reg */
1970 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1974 case 0x60: /* pusha */
1975 emulate_pusha(ctxt);
1977 case 0x61: /* popa */
1978 rc = emulate_popa(ctxt, ops);
1982 case 0x63: /* movsxd */
1983 if (ctxt->mode != X86EMUL_MODE_PROT64)
1984 goto cannot_emulate;
1985 c->dst.val = (s32) c->src.val;
1987 case 0x68: /* push imm */
1988 case 0x6a: /* push imm8 */
1991 case 0x6c: /* insb */
1992 case 0x6d: /* insw/insd */
1993 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
1994 (c->d & ByteOp) ? 1 : c->op_bytes)) {
1995 kvm_inject_gp(ctxt->vcpu, 0);
1998 if (kvm_emulate_pio_string(ctxt->vcpu,
2000 (c->d & ByteOp) ? 1 : c->op_bytes,
2002 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2003 (ctxt->eflags & EFLG_DF),
2004 register_address(c, es_base(ctxt),
2005 c->regs[VCPU_REGS_RDI]),
2007 c->regs[VCPU_REGS_RDX]) == 0) {
2012 case 0x6e: /* outsb */
2013 case 0x6f: /* outsw/outsd */
2014 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2015 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2016 kvm_inject_gp(ctxt->vcpu, 0);
2019 if (kvm_emulate_pio_string(ctxt->vcpu,
2021 (c->d & ByteOp) ? 1 : c->op_bytes,
2023 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2024 (ctxt->eflags & EFLG_DF),
2026 seg_override_base(ctxt, c),
2027 c->regs[VCPU_REGS_RSI]),
2029 c->regs[VCPU_REGS_RDX]) == 0) {
2034 case 0x70 ... 0x7f: /* jcc (short) */
2035 if (test_cc(c->b, ctxt->eflags))
2036 jmp_rel(c, c->src.val);
2038 case 0x80 ... 0x83: /* Grp1 */
2039 switch (c->modrm_reg) {
2059 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2061 case 0x86 ... 0x87: /* xchg */
2063 /* Write back the register source. */
2064 switch (c->dst.bytes) {
2066 *(u8 *) c->src.ptr = (u8) c->dst.val;
2069 *(u16 *) c->src.ptr = (u16) c->dst.val;
2072 *c->src.ptr = (u32) c->dst.val;
2073 break; /* 64b reg: zero-extend */
2075 *c->src.ptr = c->dst.val;
2079 * Write back the memory destination with implicit LOCK
2082 c->dst.val = c->src.val;
2085 case 0x88 ... 0x8b: /* mov */
2087 case 0x8c: { /* mov r/m, sreg */
2088 struct kvm_segment segreg;
2090 if (c->modrm_reg <= 5)
2091 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
2093 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
2095 goto cannot_emulate;
2097 c->dst.val = segreg.selector;
2100 case 0x8d: /* lea r16/r32, m */
2101 c->dst.val = c->modrm_ea;
2103 case 0x8e: { /* mov seg, r/m16 */
2109 if (c->modrm_reg == VCPU_SREG_SS)
2110 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
2112 if (c->modrm_reg <= 5) {
2113 type_bits = (c->modrm_reg == 1) ? 9 : 1;
2114 err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
2115 type_bits, c->modrm_reg);
2117 printk(KERN_INFO "Invalid segreg in modrm byte 0x%02x\n",
2119 goto cannot_emulate;
2123 goto cannot_emulate;
2125 c->dst.type = OP_NONE; /* Disable writeback. */
2128 case 0x8f: /* pop (sole member of Grp1a) */
2129 rc = emulate_grp1a(ctxt, ops);
2133 case 0x90: /* nop / xchg r8,rax */
2134 if (!(c->rex_prefix & 1)) { /* nop */
2135 c->dst.type = OP_NONE;
2138 case 0x91 ... 0x97: /* xchg reg,rax */
2139 c->src.type = c->dst.type = OP_REG;
2140 c->src.bytes = c->dst.bytes = c->op_bytes;
2141 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2142 c->src.val = *(c->src.ptr);
2144 case 0x9c: /* pushf */
2145 c->src.val = (unsigned long) ctxt->eflags;
2148 case 0x9d: /* popf */
2149 c->dst.type = OP_REG;
2150 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2151 c->dst.bytes = c->op_bytes;
2152 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2153 if (rc != X86EMUL_CONTINUE)
2156 case 0xa0 ... 0xa1: /* mov */
2157 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2158 c->dst.val = c->src.val;
2160 case 0xa2 ... 0xa3: /* mov */
2161 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2163 case 0xa4 ... 0xa5: /* movs */
2164 c->dst.type = OP_MEM;
2165 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2166 c->dst.ptr = (unsigned long *)register_address(c,
2168 c->regs[VCPU_REGS_RDI]);
2169 rc = ops->read_emulated(register_address(c,
2170 seg_override_base(ctxt, c),
2171 c->regs[VCPU_REGS_RSI]),
2173 c->dst.bytes, ctxt->vcpu);
2174 if (rc != X86EMUL_CONTINUE)
2176 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2177 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2179 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2180 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2183 case 0xa6 ... 0xa7: /* cmps */
2184 c->src.type = OP_NONE; /* Disable writeback. */
2185 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2186 c->src.ptr = (unsigned long *)register_address(c,
2187 seg_override_base(ctxt, c),
2188 c->regs[VCPU_REGS_RSI]);
2189 rc = ops->read_emulated((unsigned long)c->src.ptr,
2193 if (rc != X86EMUL_CONTINUE)
2196 c->dst.type = OP_NONE; /* Disable writeback. */
2197 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2198 c->dst.ptr = (unsigned long *)register_address(c,
2200 c->regs[VCPU_REGS_RDI]);
2201 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2205 if (rc != X86EMUL_CONTINUE)
2208 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2210 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2212 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2213 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2215 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2216 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2220 case 0xaa ... 0xab: /* stos */
2221 c->dst.type = OP_MEM;
2222 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2223 c->dst.ptr = (unsigned long *)register_address(c,
2225 c->regs[VCPU_REGS_RDI]);
2226 c->dst.val = c->regs[VCPU_REGS_RAX];
2227 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2228 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2231 case 0xac ... 0xad: /* lods */
2232 c->dst.type = OP_REG;
2233 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2234 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2235 rc = ops->read_emulated(register_address(c,
2236 seg_override_base(ctxt, c),
2237 c->regs[VCPU_REGS_RSI]),
2241 if (rc != X86EMUL_CONTINUE)
2243 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2244 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2247 case 0xae ... 0xaf: /* scas */
2248 DPRINTF("Urk! I don't handle SCAS.\n");
2249 goto cannot_emulate;
2250 case 0xb0 ... 0xbf: /* mov r, imm */
2255 case 0xc3: /* ret */
2256 c->dst.type = OP_REG;
2257 c->dst.ptr = &c->eip;
2258 c->dst.bytes = c->op_bytes;
2259 goto pop_instruction;
2260 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2262 c->dst.val = c->src.val;
2264 case 0xcb: /* ret far */
2265 rc = emulate_ret_far(ctxt, ops);
2269 case 0xd0 ... 0xd1: /* Grp2 */
2273 case 0xd2 ... 0xd3: /* Grp2 */
2274 c->src.val = c->regs[VCPU_REGS_RCX];
2277 case 0xe4: /* inb */
2282 case 0xe6: /* outb */
2283 case 0xe7: /* out */
2287 case 0xe8: /* call (near) */ {
2288 long int rel = c->src.val;
2289 c->src.val = (unsigned long) c->eip;
2294 case 0xe9: /* jmp rel */
2296 case 0xea: /* jmp far */
2297 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val, 9,
2298 VCPU_SREG_CS) < 0) {
2299 DPRINTF("jmp far: Failed to load CS descriptor\n");
2300 goto cannot_emulate;
2303 c->eip = c->src.val;
2306 jmp: /* jmp rel short */
2307 jmp_rel(c, c->src.val);
2308 c->dst.type = OP_NONE; /* Disable writeback. */
2310 case 0xec: /* in al,dx */
2311 case 0xed: /* in (e/r)ax,dx */
2312 port = c->regs[VCPU_REGS_RDX];
2315 case 0xee: /* out al,dx */
2316 case 0xef: /* out (e/r)ax,dx */
2317 port = c->regs[VCPU_REGS_RDX];
2320 if (!emulator_io_permited(ctxt, ops, port,
2321 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2322 kvm_inject_gp(ctxt->vcpu, 0);
2325 if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2326 (c->d & ByteOp) ? 1 : c->op_bytes,
2329 goto cannot_emulate;
2332 case 0xf4: /* hlt */
2333 ctxt->vcpu->arch.halt_request = 1;
2335 case 0xf5: /* cmc */
2336 /* complement carry flag from eflags reg */
2337 ctxt->eflags ^= EFLG_CF;
2338 c->dst.type = OP_NONE; /* Disable writeback. */
2340 case 0xf6 ... 0xf7: /* Grp3 */
2341 rc = emulate_grp3(ctxt, ops);
2345 case 0xf8: /* clc */
2346 ctxt->eflags &= ~EFLG_CF;
2347 c->dst.type = OP_NONE; /* Disable writeback. */
2349 case 0xfa: /* cli */
2350 if (emulator_bad_iopl(ctxt))
2351 kvm_inject_gp(ctxt->vcpu, 0);
2353 ctxt->eflags &= ~X86_EFLAGS_IF;
2354 c->dst.type = OP_NONE; /* Disable writeback. */
2357 case 0xfb: /* sti */
2358 if (emulator_bad_iopl(ctxt))
2359 kvm_inject_gp(ctxt->vcpu, 0);
2361 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2362 ctxt->eflags |= X86_EFLAGS_IF;
2363 c->dst.type = OP_NONE; /* Disable writeback. */
2366 case 0xfc: /* cld */
2367 ctxt->eflags &= ~EFLG_DF;
2368 c->dst.type = OP_NONE; /* Disable writeback. */
2370 case 0xfd: /* std */
2371 ctxt->eflags |= EFLG_DF;
2372 c->dst.type = OP_NONE; /* Disable writeback. */
2374 case 0xfe ... 0xff: /* Grp4/Grp5 */
2375 rc = emulate_grp45(ctxt, ops);
2382 rc = writeback(ctxt, ops);
2386 /* Commit shadow register state. */
2387 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2388 kvm_rip_write(ctxt->vcpu, c->eip);
2391 if (rc == X86EMUL_UNHANDLEABLE) {
2399 case 0x01: /* lgdt, lidt, lmsw */
2400 switch (c->modrm_reg) {
2402 unsigned long address;
2404 case 0: /* vmcall */
2405 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2406 goto cannot_emulate;
2408 rc = kvm_fix_hypercall(ctxt->vcpu);
2412 /* Let the processor re-execute the fixed hypercall */
2413 c->eip = kvm_rip_read(ctxt->vcpu);
2414 /* Disable writeback. */
2415 c->dst.type = OP_NONE;
2418 rc = read_descriptor(ctxt, ops, c->src.ptr,
2419 &size, &address, c->op_bytes);
2422 realmode_lgdt(ctxt->vcpu, size, address);
2423 /* Disable writeback. */
2424 c->dst.type = OP_NONE;
2426 case 3: /* lidt/vmmcall */
2427 if (c->modrm_mod == 3) {
2428 switch (c->modrm_rm) {
2430 rc = kvm_fix_hypercall(ctxt->vcpu);
2435 goto cannot_emulate;
2438 rc = read_descriptor(ctxt, ops, c->src.ptr,
2443 realmode_lidt(ctxt->vcpu, size, address);
2445 /* Disable writeback. */
2446 c->dst.type = OP_NONE;
2450 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2453 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2455 c->dst.type = OP_NONE;
2458 emulate_invlpg(ctxt->vcpu, memop);
2459 /* Disable writeback. */
2460 c->dst.type = OP_NONE;
2463 goto cannot_emulate;
2466 case 0x05: /* syscall */
2467 if (emulate_syscall(ctxt) == -1)
2468 goto cannot_emulate;
2473 emulate_clts(ctxt->vcpu);
2474 c->dst.type = OP_NONE;
2476 case 0x08: /* invd */
2477 case 0x09: /* wbinvd */
2478 case 0x0d: /* GrpP (prefetch) */
2479 case 0x18: /* Grp16 (prefetch/nop) */
2480 c->dst.type = OP_NONE;
2482 case 0x20: /* mov cr, reg */
2483 if (c->modrm_mod != 3)
2484 goto cannot_emulate;
2485 c->regs[c->modrm_rm] =
2486 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2487 c->dst.type = OP_NONE; /* no writeback */
2489 case 0x21: /* mov from dr to reg */
2490 if (c->modrm_mod != 3)
2491 goto cannot_emulate;
2492 rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
2494 goto cannot_emulate;
2495 c->dst.type = OP_NONE; /* no writeback */
2497 case 0x22: /* mov reg, cr */
2498 if (c->modrm_mod != 3)
2499 goto cannot_emulate;
2500 realmode_set_cr(ctxt->vcpu,
2501 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2502 c->dst.type = OP_NONE;
2504 case 0x23: /* mov from reg to dr */
2505 if (c->modrm_mod != 3)
2506 goto cannot_emulate;
2507 rc = emulator_set_dr(ctxt, c->modrm_reg,
2508 c->regs[c->modrm_rm]);
2510 goto cannot_emulate;
2511 c->dst.type = OP_NONE; /* no writeback */
2515 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2516 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2517 rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
2519 kvm_inject_gp(ctxt->vcpu, 0);
2520 c->eip = kvm_rip_read(ctxt->vcpu);
2522 rc = X86EMUL_CONTINUE;
2523 c->dst.type = OP_NONE;
2527 rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
2529 kvm_inject_gp(ctxt->vcpu, 0);
2530 c->eip = kvm_rip_read(ctxt->vcpu);
2532 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2533 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2535 rc = X86EMUL_CONTINUE;
2536 c->dst.type = OP_NONE;
2538 case 0x34: /* sysenter */
2539 if (emulate_sysenter(ctxt) == -1)
2540 goto cannot_emulate;
2544 case 0x35: /* sysexit */
2545 if (emulate_sysexit(ctxt) == -1)
2546 goto cannot_emulate;
2550 case 0x40 ... 0x4f: /* cmov */
2551 c->dst.val = c->dst.orig_val = c->src.val;
2552 if (!test_cc(c->b, ctxt->eflags))
2553 c->dst.type = OP_NONE; /* no writeback */
2555 case 0x80 ... 0x8f: /* jnz rel, etc*/
2556 if (test_cc(c->b, ctxt->eflags))
2557 jmp_rel(c, c->src.val);
2558 c->dst.type = OP_NONE;
2560 case 0xa0: /* push fs */
2561 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2563 case 0xa1: /* pop fs */
2564 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2570 c->dst.type = OP_NONE;
2571 /* only subword offset */
2572 c->src.val &= (c->dst.bytes << 3) - 1;
2573 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2575 case 0xa4: /* shld imm8, r, r/m */
2576 case 0xa5: /* shld cl, r, r/m */
2577 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2579 case 0xa8: /* push gs */
2580 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2582 case 0xa9: /* pop gs */
2583 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2589 /* only subword offset */
2590 c->src.val &= (c->dst.bytes << 3) - 1;
2591 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2593 case 0xac: /* shrd imm8, r, r/m */
2594 case 0xad: /* shrd cl, r, r/m */
2595 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2597 case 0xae: /* clflush */
2599 case 0xb0 ... 0xb1: /* cmpxchg */
2601 * Save real source value, then compare EAX against
2604 c->src.orig_val = c->src.val;
2605 c->src.val = c->regs[VCPU_REGS_RAX];
2606 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2607 if (ctxt->eflags & EFLG_ZF) {
2608 /* Success: write back to memory. */
2609 c->dst.val = c->src.orig_val;
2611 /* Failure: write the value we saw to EAX. */
2612 c->dst.type = OP_REG;
2613 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2618 /* only subword offset */
2619 c->src.val &= (c->dst.bytes << 3) - 1;
2620 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2622 case 0xb6 ... 0xb7: /* movzx */
2623 c->dst.bytes = c->op_bytes;
2624 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2627 case 0xba: /* Grp8 */
2628 switch (c->modrm_reg & 3) {
2641 /* only subword offset */
2642 c->src.val &= (c->dst.bytes << 3) - 1;
2643 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2645 case 0xbe ... 0xbf: /* movsx */
2646 c->dst.bytes = c->op_bytes;
2647 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2650 case 0xc3: /* movnti */
2651 c->dst.bytes = c->op_bytes;
2652 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2655 case 0xc7: /* Grp9 (cmpxchg8b) */
2656 rc = emulate_grp9(ctxt, ops, memop);
2659 c->dst.type = OP_NONE;
2665 DPRINTF("Cannot emulate %02x\n", c->b);