1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
36 #include "mmu.h" /* for is_long_mode() */
39 * Opcode effective-address decode tables.
40 * Note that we only emulate instructions that have at least one memory
41 * operand (excluding implicit stack references). We assume that stack
42 * references and instruction fetches will never occur in special memory
43 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
47 /* Operand sizes: 8-bit operands or specified/overridden size. */
48 #define ByteOp (1<<0) /* 8-bit operands. */
49 /* Destination operand type. */
50 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
51 #define DstReg (2<<1) /* Register operand. */
52 #define DstMem (3<<1) /* Memory operand. */
53 #define DstAcc (4<<1) /* Destination Accumulator */
54 #define DstMask (7<<1)
55 /* Source operand type. */
56 #define SrcNone (0<<4) /* No source operand. */
57 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
58 #define SrcReg (1<<4) /* Register operand. */
59 #define SrcMem (2<<4) /* Memory operand. */
60 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
61 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
62 #define SrcImm (5<<4) /* Immediate operand. */
63 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
64 #define SrcOne (7<<4) /* Implied '1' */
65 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
66 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define GroupMask 0xff /* Group number stored in bits 0:7 */
81 /* Source 2 operand type */
82 #define Src2None (0<<29)
83 #define Src2CL (1<<29)
84 #define Src2ImmByte (2<<29)
85 #define Src2One (3<<29)
86 #define Src2Imm16 (4<<29)
87 #define Src2Mask (7<<29)
90 Group1_80, Group1_81, Group1_82, Group1_83,
91 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
94 static u32 opcode_table[256] = {
96 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
97 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
98 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
99 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
101 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
102 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
103 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
104 ImplicitOps | Stack | No64, 0,
106 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
107 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
108 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
109 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
111 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
112 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
113 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
114 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
116 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
117 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
118 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
120 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
121 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
124 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
125 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
128 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
129 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
133 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
135 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
137 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
138 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
140 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
141 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
143 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
144 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
147 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
148 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
149 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
151 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
152 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
155 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 Group | Group1_80, Group | Group1_81,
158 Group | Group1_82, Group | Group1_83,
159 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
160 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
162 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
163 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
164 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
165 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
167 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
169 0, 0, SrcImm | Src2Imm16 | No64, 0,
170 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
172 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
173 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
174 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
175 ByteOp | ImplicitOps | String, ImplicitOps | String,
177 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
178 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
179 ByteOp | ImplicitOps | String, ImplicitOps | String,
181 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
182 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
184 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
186 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
187 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
189 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
191 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
192 0, ImplicitOps | Stack, 0, 0,
193 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
195 0, 0, 0, ImplicitOps | Stack,
196 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
198 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
199 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
202 0, 0, 0, 0, 0, 0, 0, 0,
205 ByteOp | SrcImmUByte, SrcImmUByte,
206 ByteOp | SrcImmUByte, SrcImmUByte,
208 SrcImm | Stack, SrcImm | ImplicitOps,
209 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
210 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
211 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
214 ImplicitOps, ImplicitOps, Group | Group3_Byte, Group | Group3,
216 ImplicitOps, 0, ImplicitOps, ImplicitOps,
217 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
220 static u32 twobyte_table[256] = {
222 0, Group | GroupDual | Group7, 0, 0, 0, ImplicitOps, ImplicitOps, 0,
223 ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0,
225 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
227 ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0,
228 0, 0, 0, 0, 0, 0, 0, 0,
230 ImplicitOps, 0, ImplicitOps, 0,
231 ImplicitOps, ImplicitOps, 0, 0,
232 0, 0, 0, 0, 0, 0, 0, 0,
234 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
235 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
236 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
237 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
239 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
244 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
246 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
248 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
250 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
251 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
253 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
255 ImplicitOps | Stack, ImplicitOps | Stack,
256 0, DstMem | SrcReg | ModRM | BitOp,
257 DstMem | SrcReg | Src2ImmByte | ModRM,
258 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
260 ImplicitOps | Stack, ImplicitOps | Stack,
261 0, DstMem | SrcReg | ModRM | BitOp,
262 DstMem | SrcReg | Src2ImmByte | ModRM,
263 DstMem | SrcReg | Src2CL | ModRM,
266 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0,
267 DstMem | SrcReg | ModRM | BitOp,
268 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
269 DstReg | SrcMem16 | ModRM | Mov,
271 0, 0, DstMem | SrcImmByte | ModRM, DstMem | SrcReg | ModRM | BitOp,
272 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
273 DstReg | SrcMem16 | ModRM | Mov,
275 0, 0, 0, DstMem | SrcReg | ModRM | Mov, 0, 0, 0, ImplicitOps | ModRM,
276 0, 0, 0, 0, 0, 0, 0, 0,
278 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
280 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
282 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
285 static u32 group_table[] = {
287 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
288 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
289 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
290 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
292 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
293 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
294 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
295 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
297 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
298 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
299 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
300 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
302 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
303 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
304 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
305 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
307 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
309 ByteOp | SrcImm | DstMem | ModRM, 0,
310 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
313 DstMem | SrcImm | ModRM, 0,
314 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
317 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
320 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
321 SrcMem | ModRM | Stack, 0,
322 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
324 0, 0, ModRM | SrcMem, ModRM | SrcMem,
325 SrcNone | ModRM | DstMem | Mov, 0,
326 SrcMem16 | ModRM | Mov, SrcMem | ModRM | ByteOp,
329 static u32 group2_table[] = {
331 SrcNone | ModRM, 0, 0, SrcNone | ModRM,
332 SrcNone | ModRM | DstMem | Mov, 0,
333 SrcMem16 | ModRM | Mov, 0,
336 /* EFLAGS bit definitions. */
337 #define EFLG_VM (1<<17)
338 #define EFLG_RF (1<<16)
339 #define EFLG_OF (1<<11)
340 #define EFLG_DF (1<<10)
341 #define EFLG_IF (1<<9)
342 #define EFLG_SF (1<<7)
343 #define EFLG_ZF (1<<6)
344 #define EFLG_AF (1<<4)
345 #define EFLG_PF (1<<2)
346 #define EFLG_CF (1<<0)
349 * Instruction emulation:
350 * Most instructions are emulated directly via a fragment of inline assembly
351 * code. This allows us to save/restore EFLAGS and thus very easily pick up
352 * any modified flags.
355 #if defined(CONFIG_X86_64)
356 #define _LO32 "k" /* force 32-bit operand */
357 #define _STK "%%rsp" /* stack pointer */
358 #elif defined(__i386__)
359 #define _LO32 "" /* force 32-bit operand */
360 #define _STK "%%esp" /* stack pointer */
364 * These EFLAGS bits are restored from saved value during emulation, and
365 * any changes are written back to the saved value after emulation.
367 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
369 /* Before executing instruction: restore necessary bits in EFLAGS. */
370 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
371 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
372 "movl %"_sav",%"_LO32 _tmp"; " \
375 "movl %"_msk",%"_LO32 _tmp"; " \
376 "andl %"_LO32 _tmp",("_STK"); " \
378 "notl %"_LO32 _tmp"; " \
379 "andl %"_LO32 _tmp",("_STK"); " \
380 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
382 "orl %"_LO32 _tmp",("_STK"); " \
386 /* After executing instruction: write-back necessary bits in EFLAGS. */
387 #define _POST_EFLAGS(_sav, _msk, _tmp) \
388 /* _sav |= EFLAGS & _msk; */ \
391 "andl %"_msk",%"_LO32 _tmp"; " \
392 "orl %"_LO32 _tmp",%"_sav"; "
400 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
402 __asm__ __volatile__ ( \
403 _PRE_EFLAGS("0", "4", "2") \
404 _op _suffix " %"_x"3,%1; " \
405 _POST_EFLAGS("0", "4", "2") \
406 : "=m" (_eflags), "=m" ((_dst).val), \
408 : _y ((_src).val), "i" (EFLAGS_MASK)); \
412 /* Raw emulation: instruction has two explicit operands. */
413 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
415 unsigned long _tmp; \
417 switch ((_dst).bytes) { \
419 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
422 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
425 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
430 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
432 unsigned long _tmp; \
433 switch ((_dst).bytes) { \
435 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
438 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
439 _wx, _wy, _lx, _ly, _qx, _qy); \
444 /* Source operand is byte-sized and may be restricted to just %cl. */
445 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
446 __emulate_2op(_op, _src, _dst, _eflags, \
447 "b", "c", "b", "c", "b", "c", "b", "c")
449 /* Source operand is byte, word, long or quad sized. */
450 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
451 __emulate_2op(_op, _src, _dst, _eflags, \
452 "b", "q", "w", "r", _LO32, "r", "", "r")
454 /* Source operand is word, long or quad sized. */
455 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
456 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
457 "w", "r", _LO32, "r", "", "r")
459 /* Instruction has three operands and one operand is stored in ECX register */
460 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
462 unsigned long _tmp; \
463 _type _clv = (_cl).val; \
464 _type _srcv = (_src).val; \
465 _type _dstv = (_dst).val; \
467 __asm__ __volatile__ ( \
468 _PRE_EFLAGS("0", "5", "2") \
469 _op _suffix " %4,%1 \n" \
470 _POST_EFLAGS("0", "5", "2") \
471 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
472 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
475 (_cl).val = (unsigned long) _clv; \
476 (_src).val = (unsigned long) _srcv; \
477 (_dst).val = (unsigned long) _dstv; \
480 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
482 switch ((_dst).bytes) { \
484 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
485 "w", unsigned short); \
488 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
489 "l", unsigned int); \
492 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
493 "q", unsigned long)); \
498 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
500 unsigned long _tmp; \
502 __asm__ __volatile__ ( \
503 _PRE_EFLAGS("0", "3", "2") \
504 _op _suffix " %1; " \
505 _POST_EFLAGS("0", "3", "2") \
506 : "=m" (_eflags), "+m" ((_dst).val), \
508 : "i" (EFLAGS_MASK)); \
511 /* Instruction has only one explicit operand (no source operand). */
512 #define emulate_1op(_op, _dst, _eflags) \
514 switch ((_dst).bytes) { \
515 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
516 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
517 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
518 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
522 /* Fetch next part of the instruction being emulated. */
523 #define insn_fetch(_type, _size, _eip) \
524 ({ unsigned long _x; \
525 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
532 static inline unsigned long ad_mask(struct decode_cache *c)
534 return (1UL << (c->ad_bytes << 3)) - 1;
537 /* Access/update address held in a register, based on addressing mode. */
538 static inline unsigned long
539 address_mask(struct decode_cache *c, unsigned long reg)
541 if (c->ad_bytes == sizeof(unsigned long))
544 return reg & ad_mask(c);
547 static inline unsigned long
548 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
550 return base + address_mask(c, reg);
554 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
556 if (c->ad_bytes == sizeof(unsigned long))
559 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
562 static inline void jmp_rel(struct decode_cache *c, int rel)
564 register_address_increment(c, &c->eip, rel);
567 static void set_seg_override(struct decode_cache *c, int seg)
569 c->has_seg_override = true;
570 c->seg_override = seg;
573 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
575 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
578 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
581 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
582 struct decode_cache *c)
584 if (!c->has_seg_override)
587 return seg_base(ctxt, c->seg_override);
590 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
592 return seg_base(ctxt, VCPU_SREG_ES);
595 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
597 return seg_base(ctxt, VCPU_SREG_SS);
600 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
601 struct x86_emulate_ops *ops,
602 unsigned long linear, u8 *dest)
604 struct fetch_cache *fc = &ctxt->decode.fetch;
608 if (linear < fc->start || linear >= fc->end) {
609 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
610 rc = ops->read_std(linear, fc->data, size, ctxt->vcpu);
614 fc->end = linear + size;
616 *dest = fc->data[linear - fc->start];
620 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
621 struct x86_emulate_ops *ops,
622 unsigned long eip, void *dest, unsigned size)
626 /* x86 instructions are limited to 15 bytes. */
627 if (eip + size - ctxt->decode.eip_orig > 15)
628 return X86EMUL_UNHANDLEABLE;
629 eip += ctxt->cs_base;
631 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
639 * Given the 'reg' portion of a ModRM byte, and a register block, return a
640 * pointer into the block that addresses the relevant register.
641 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
643 static void *decode_register(u8 modrm_reg, unsigned long *regs,
648 p = ®s[modrm_reg];
649 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
650 p = (unsigned char *)®s[modrm_reg & 3] + 1;
654 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
655 struct x86_emulate_ops *ops,
657 u16 *size, unsigned long *address, int op_bytes)
664 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
668 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
673 static int test_cc(unsigned int condition, unsigned int flags)
677 switch ((condition & 15) >> 1) {
679 rc |= (flags & EFLG_OF);
681 case 1: /* b/c/nae */
682 rc |= (flags & EFLG_CF);
685 rc |= (flags & EFLG_ZF);
688 rc |= (flags & (EFLG_CF|EFLG_ZF));
691 rc |= (flags & EFLG_SF);
694 rc |= (flags & EFLG_PF);
697 rc |= (flags & EFLG_ZF);
700 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
704 /* Odd condition identifiers (lsb == 1) have inverted sense. */
705 return (!!rc ^ (condition & 1));
708 static void decode_register_operand(struct operand *op,
709 struct decode_cache *c,
712 unsigned reg = c->modrm_reg;
713 int highbyte_regs = c->rex_prefix == 0;
716 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
718 if ((c->d & ByteOp) && !inhibit_bytereg) {
719 op->ptr = decode_register(reg, c->regs, highbyte_regs);
720 op->val = *(u8 *)op->ptr;
723 op->ptr = decode_register(reg, c->regs, 0);
724 op->bytes = c->op_bytes;
727 op->val = *(u16 *)op->ptr;
730 op->val = *(u32 *)op->ptr;
733 op->val = *(u64 *) op->ptr;
737 op->orig_val = op->val;
740 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
741 struct x86_emulate_ops *ops)
743 struct decode_cache *c = &ctxt->decode;
745 int index_reg = 0, base_reg = 0, scale;
749 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
750 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
751 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
754 c->modrm = insn_fetch(u8, 1, c->eip);
755 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
756 c->modrm_reg |= (c->modrm & 0x38) >> 3;
757 c->modrm_rm |= (c->modrm & 0x07);
761 if (c->modrm_mod == 3) {
762 c->modrm_ptr = decode_register(c->modrm_rm,
763 c->regs, c->d & ByteOp);
764 c->modrm_val = *(unsigned long *)c->modrm_ptr;
768 if (c->ad_bytes == 2) {
769 unsigned bx = c->regs[VCPU_REGS_RBX];
770 unsigned bp = c->regs[VCPU_REGS_RBP];
771 unsigned si = c->regs[VCPU_REGS_RSI];
772 unsigned di = c->regs[VCPU_REGS_RDI];
774 /* 16-bit ModR/M decode. */
775 switch (c->modrm_mod) {
777 if (c->modrm_rm == 6)
778 c->modrm_ea += insn_fetch(u16, 2, c->eip);
781 c->modrm_ea += insn_fetch(s8, 1, c->eip);
784 c->modrm_ea += insn_fetch(u16, 2, c->eip);
787 switch (c->modrm_rm) {
789 c->modrm_ea += bx + si;
792 c->modrm_ea += bx + di;
795 c->modrm_ea += bp + si;
798 c->modrm_ea += bp + di;
807 if (c->modrm_mod != 0)
814 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
815 (c->modrm_rm == 6 && c->modrm_mod != 0))
816 if (!c->has_seg_override)
817 set_seg_override(c, VCPU_SREG_SS);
818 c->modrm_ea = (u16)c->modrm_ea;
820 /* 32/64-bit ModR/M decode. */
821 if ((c->modrm_rm & 7) == 4) {
822 sib = insn_fetch(u8, 1, c->eip);
823 index_reg |= (sib >> 3) & 7;
827 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
828 c->modrm_ea += insn_fetch(s32, 4, c->eip);
830 c->modrm_ea += c->regs[base_reg];
832 c->modrm_ea += c->regs[index_reg] << scale;
833 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
834 if (ctxt->mode == X86EMUL_MODE_PROT64)
837 c->modrm_ea += c->regs[c->modrm_rm];
838 switch (c->modrm_mod) {
840 if (c->modrm_rm == 5)
841 c->modrm_ea += insn_fetch(s32, 4, c->eip);
844 c->modrm_ea += insn_fetch(s8, 1, c->eip);
847 c->modrm_ea += insn_fetch(s32, 4, c->eip);
855 static int decode_abs(struct x86_emulate_ctxt *ctxt,
856 struct x86_emulate_ops *ops)
858 struct decode_cache *c = &ctxt->decode;
861 switch (c->ad_bytes) {
863 c->modrm_ea = insn_fetch(u16, 2, c->eip);
866 c->modrm_ea = insn_fetch(u32, 4, c->eip);
869 c->modrm_ea = insn_fetch(u64, 8, c->eip);
877 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
879 struct decode_cache *c = &ctxt->decode;
881 int mode = ctxt->mode;
882 int def_op_bytes, def_ad_bytes, group;
884 /* Shadow copy of register state. Committed on successful emulation. */
886 memset(c, 0, sizeof(struct decode_cache));
887 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
888 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
889 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
892 case X86EMUL_MODE_REAL:
893 case X86EMUL_MODE_PROT16:
894 def_op_bytes = def_ad_bytes = 2;
896 case X86EMUL_MODE_PROT32:
897 def_op_bytes = def_ad_bytes = 4;
900 case X86EMUL_MODE_PROT64:
909 c->op_bytes = def_op_bytes;
910 c->ad_bytes = def_ad_bytes;
912 /* Legacy prefixes. */
914 switch (c->b = insn_fetch(u8, 1, c->eip)) {
915 case 0x66: /* operand-size override */
916 /* switch between 2/4 bytes */
917 c->op_bytes = def_op_bytes ^ 6;
919 case 0x67: /* address-size override */
920 if (mode == X86EMUL_MODE_PROT64)
921 /* switch between 4/8 bytes */
922 c->ad_bytes = def_ad_bytes ^ 12;
924 /* switch between 2/4 bytes */
925 c->ad_bytes = def_ad_bytes ^ 6;
927 case 0x26: /* ES override */
928 case 0x2e: /* CS override */
929 case 0x36: /* SS override */
930 case 0x3e: /* DS override */
931 set_seg_override(c, (c->b >> 3) & 3);
933 case 0x64: /* FS override */
934 case 0x65: /* GS override */
935 set_seg_override(c, c->b & 7);
937 case 0x40 ... 0x4f: /* REX */
938 if (mode != X86EMUL_MODE_PROT64)
940 c->rex_prefix = c->b;
942 case 0xf0: /* LOCK */
945 case 0xf2: /* REPNE/REPNZ */
946 c->rep_prefix = REPNE_PREFIX;
948 case 0xf3: /* REP/REPE/REPZ */
949 c->rep_prefix = REPE_PREFIX;
955 /* Any legacy prefix after a REX prefix nullifies its effect. */
964 if (c->rex_prefix & 8)
965 c->op_bytes = 8; /* REX.W */
967 /* Opcode byte(s). */
968 c->d = opcode_table[c->b];
970 /* Two-byte opcode? */
973 c->b = insn_fetch(u8, 1, c->eip);
974 c->d = twobyte_table[c->b];
978 if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
979 kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");;
984 group = c->d & GroupMask;
985 c->modrm = insn_fetch(u8, 1, c->eip);
988 group = (group << 3) + ((c->modrm >> 3) & 7);
989 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
990 c->d = group2_table[group];
992 c->d = group_table[group];
997 DPRINTF("Cannot emulate %02x\n", c->b);
1001 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1004 /* ModRM and SIB bytes. */
1006 rc = decode_modrm(ctxt, ops);
1007 else if (c->d & MemAbs)
1008 rc = decode_abs(ctxt, ops);
1012 if (!c->has_seg_override)
1013 set_seg_override(c, VCPU_SREG_DS);
1015 if (!(!c->twobyte && c->b == 0x8d))
1016 c->modrm_ea += seg_override_base(ctxt, c);
1018 if (c->ad_bytes != 8)
1019 c->modrm_ea = (u32)c->modrm_ea;
1021 * Decode and fetch the source operand: register, memory
1024 switch (c->d & SrcMask) {
1028 decode_register_operand(&c->src, c, 0);
1037 c->src.bytes = (c->d & ByteOp) ? 1 :
1039 /* Don't fetch the address for invlpg: it could be unmapped. */
1040 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1044 * For instructions with a ModR/M byte, switch to register
1045 * access if Mod = 3.
1047 if ((c->d & ModRM) && c->modrm_mod == 3) {
1048 c->src.type = OP_REG;
1049 c->src.val = c->modrm_val;
1050 c->src.ptr = c->modrm_ptr;
1053 c->src.type = OP_MEM;
1057 c->src.type = OP_IMM;
1058 c->src.ptr = (unsigned long *)c->eip;
1059 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1060 if (c->src.bytes == 8)
1062 /* NB. Immediates are sign-extended as necessary. */
1063 switch (c->src.bytes) {
1065 c->src.val = insn_fetch(s8, 1, c->eip);
1068 c->src.val = insn_fetch(s16, 2, c->eip);
1071 c->src.val = insn_fetch(s32, 4, c->eip);
1074 if ((c->d & SrcMask) == SrcImmU) {
1075 switch (c->src.bytes) {
1080 c->src.val &= 0xffff;
1083 c->src.val &= 0xffffffff;
1090 c->src.type = OP_IMM;
1091 c->src.ptr = (unsigned long *)c->eip;
1093 if ((c->d & SrcMask) == SrcImmByte)
1094 c->src.val = insn_fetch(s8, 1, c->eip);
1096 c->src.val = insn_fetch(u8, 1, c->eip);
1105 * Decode and fetch the second source operand: register, memory
1108 switch (c->d & Src2Mask) {
1113 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1116 c->src2.type = OP_IMM;
1117 c->src2.ptr = (unsigned long *)c->eip;
1119 c->src2.val = insn_fetch(u8, 1, c->eip);
1122 c->src2.type = OP_IMM;
1123 c->src2.ptr = (unsigned long *)c->eip;
1125 c->src2.val = insn_fetch(u16, 2, c->eip);
1133 /* Decode and fetch the destination operand: register or memory. */
1134 switch (c->d & DstMask) {
1136 /* Special instructions do their own operand decoding. */
1139 decode_register_operand(&c->dst, c,
1140 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1143 if ((c->d & ModRM) && c->modrm_mod == 3) {
1144 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1145 c->dst.type = OP_REG;
1146 c->dst.val = c->dst.orig_val = c->modrm_val;
1147 c->dst.ptr = c->modrm_ptr;
1150 c->dst.type = OP_MEM;
1153 c->dst.type = OP_REG;
1154 c->dst.bytes = c->op_bytes;
1155 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1156 switch (c->op_bytes) {
1158 c->dst.val = *(u8 *)c->dst.ptr;
1161 c->dst.val = *(u16 *)c->dst.ptr;
1164 c->dst.val = *(u32 *)c->dst.ptr;
1167 c->dst.orig_val = c->dst.val;
1171 if (c->rip_relative)
1172 c->modrm_ea += c->eip;
1175 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1178 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1180 struct decode_cache *c = &ctxt->decode;
1182 c->dst.type = OP_MEM;
1183 c->dst.bytes = c->op_bytes;
1184 c->dst.val = c->src.val;
1185 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1186 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1187 c->regs[VCPU_REGS_RSP]);
1190 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1191 struct x86_emulate_ops *ops,
1192 void *dest, int len)
1194 struct decode_cache *c = &ctxt->decode;
1197 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1198 c->regs[VCPU_REGS_RSP]),
1199 dest, len, ctxt->vcpu);
1200 if (rc != X86EMUL_CONTINUE)
1203 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1207 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1209 struct decode_cache *c = &ctxt->decode;
1210 struct kvm_segment segment;
1212 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1214 c->src.val = segment.selector;
1218 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1219 struct x86_emulate_ops *ops, int seg)
1221 struct decode_cache *c = &ctxt->decode;
1222 unsigned long selector;
1225 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1229 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, 1, seg);
1233 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1235 struct decode_cache *c = &ctxt->decode;
1236 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1237 int reg = VCPU_REGS_RAX;
1239 while (reg <= VCPU_REGS_RDI) {
1240 (reg == VCPU_REGS_RSP) ?
1241 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1248 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1249 struct x86_emulate_ops *ops)
1251 struct decode_cache *c = &ctxt->decode;
1253 int reg = VCPU_REGS_RDI;
1255 while (reg >= VCPU_REGS_RAX) {
1256 if (reg == VCPU_REGS_RSP) {
1257 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1262 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1270 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1271 struct x86_emulate_ops *ops)
1273 struct decode_cache *c = &ctxt->decode;
1276 rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1282 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1284 struct decode_cache *c = &ctxt->decode;
1285 switch (c->modrm_reg) {
1287 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1290 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1293 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1296 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1298 case 4: /* sal/shl */
1299 case 6: /* sal/shl */
1300 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1303 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1306 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1311 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1312 struct x86_emulate_ops *ops)
1314 struct decode_cache *c = &ctxt->decode;
1317 switch (c->modrm_reg) {
1318 case 0 ... 1: /* test */
1319 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1322 c->dst.val = ~c->dst.val;
1325 emulate_1op("neg", c->dst, ctxt->eflags);
1328 DPRINTF("Cannot emulate %02x\n", c->b);
1329 rc = X86EMUL_UNHANDLEABLE;
1335 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1336 struct x86_emulate_ops *ops)
1338 struct decode_cache *c = &ctxt->decode;
1340 switch (c->modrm_reg) {
1342 emulate_1op("inc", c->dst, ctxt->eflags);
1345 emulate_1op("dec", c->dst, ctxt->eflags);
1347 case 2: /* call near abs */ {
1350 c->eip = c->src.val;
1351 c->src.val = old_eip;
1355 case 4: /* jmp abs */
1356 c->eip = c->src.val;
1365 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1366 struct x86_emulate_ops *ops,
1367 unsigned long memop)
1369 struct decode_cache *c = &ctxt->decode;
1373 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1374 if (rc != X86EMUL_CONTINUE)
1377 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1378 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1380 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1381 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1382 ctxt->eflags &= ~EFLG_ZF;
1385 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1386 (u32) c->regs[VCPU_REGS_RBX];
1388 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1389 if (rc != X86EMUL_CONTINUE)
1391 ctxt->eflags |= EFLG_ZF;
1396 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1397 struct x86_emulate_ops *ops)
1399 struct decode_cache *c = &ctxt->decode;
1403 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1406 if (c->op_bytes == 4)
1407 c->eip = (u32)c->eip;
1408 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1411 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, 1, VCPU_SREG_CS);
1415 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1416 struct x86_emulate_ops *ops)
1419 struct decode_cache *c = &ctxt->decode;
1421 switch (c->dst.type) {
1423 /* The 4-byte case *is* correct:
1424 * in 64-bit mode we zero-extend.
1426 switch (c->dst.bytes) {
1428 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1431 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1434 *c->dst.ptr = (u32)c->dst.val;
1435 break; /* 64b: zero-ext */
1437 *c->dst.ptr = c->dst.val;
1443 rc = ops->cmpxchg_emulated(
1444 (unsigned long)c->dst.ptr,
1450 rc = ops->write_emulated(
1451 (unsigned long)c->dst.ptr,
1455 if (rc != X86EMUL_CONTINUE)
1467 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1469 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1471 * an sti; sti; sequence only disable interrupts for the first
1472 * instruction. So, if the last instruction, be it emulated or
1473 * not, left the system with the INT_STI flag enabled, it
1474 * means that the last instruction is an sti. We should not
1475 * leave the flag on in this case. The same goes for mov ss
1477 if (!(int_shadow & mask))
1478 ctxt->interruptibility = mask;
1482 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1483 struct kvm_segment *cs, struct kvm_segment *ss)
1485 memset(cs, 0, sizeof(struct kvm_segment));
1486 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1487 memset(ss, 0, sizeof(struct kvm_segment));
1489 cs->l = 0; /* will be adjusted later */
1490 cs->base = 0; /* flat segment */
1491 cs->g = 1; /* 4kb granularity */
1492 cs->limit = 0xffffffff; /* 4GB limit */
1493 cs->type = 0x0b; /* Read, Execute, Accessed */
1495 cs->dpl = 0; /* will be adjusted later */
1500 ss->base = 0; /* flat segment */
1501 ss->limit = 0xffffffff; /* 4GB limit */
1502 ss->g = 1; /* 4kb granularity */
1504 ss->type = 0x03; /* Read/Write, Accessed */
1505 ss->db = 1; /* 32bit stack segment */
1511 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1513 struct decode_cache *c = &ctxt->decode;
1514 struct kvm_segment cs, ss;
1517 /* syscall is not available in real mode */
1518 if (c->lock_prefix || ctxt->mode == X86EMUL_MODE_REAL
1519 || !is_protmode(ctxt->vcpu))
1522 setup_syscalls_segments(ctxt, &cs, &ss);
1524 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1526 cs.selector = (u16)(msr_data & 0xfffc);
1527 ss.selector = (u16)(msr_data + 8);
1529 if (is_long_mode(ctxt->vcpu)) {
1533 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1534 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1536 c->regs[VCPU_REGS_RCX] = c->eip;
1537 if (is_long_mode(ctxt->vcpu)) {
1538 #ifdef CONFIG_X86_64
1539 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1541 kvm_x86_ops->get_msr(ctxt->vcpu,
1542 ctxt->mode == X86EMUL_MODE_PROT64 ?
1543 MSR_LSTAR : MSR_CSTAR, &msr_data);
1546 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1547 ctxt->eflags &= ~(msr_data | EFLG_RF);
1551 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1552 c->eip = (u32)msr_data;
1554 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1561 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1563 struct decode_cache *c = &ctxt->decode;
1564 struct kvm_segment cs, ss;
1567 /* inject #UD if LOCK prefix is used */
1571 /* inject #GP if in real mode or paging is disabled */
1572 if (ctxt->mode == X86EMUL_MODE_REAL || !is_protmode(ctxt->vcpu)) {
1573 kvm_inject_gp(ctxt->vcpu, 0);
1577 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1578 * Therefore, we inject an #UD.
1580 if (ctxt->mode == X86EMUL_MODE_PROT64)
1583 setup_syscalls_segments(ctxt, &cs, &ss);
1585 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1586 switch (ctxt->mode) {
1587 case X86EMUL_MODE_PROT32:
1588 if ((msr_data & 0xfffc) == 0x0) {
1589 kvm_inject_gp(ctxt->vcpu, 0);
1593 case X86EMUL_MODE_PROT64:
1594 if (msr_data == 0x0) {
1595 kvm_inject_gp(ctxt->vcpu, 0);
1601 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1602 cs.selector = (u16)msr_data;
1603 cs.selector &= ~SELECTOR_RPL_MASK;
1604 ss.selector = cs.selector + 8;
1605 ss.selector &= ~SELECTOR_RPL_MASK;
1606 if (ctxt->mode == X86EMUL_MODE_PROT64
1607 || is_long_mode(ctxt->vcpu)) {
1612 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1613 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1615 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1618 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1619 c->regs[VCPU_REGS_RSP] = msr_data;
1625 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1627 struct decode_cache *c = &ctxt->decode;
1628 struct kvm_segment cs, ss;
1632 /* inject #UD if LOCK prefix is used */
1636 /* inject #GP if in real mode or paging is disabled */
1637 if (ctxt->mode == X86EMUL_MODE_REAL || !is_protmode(ctxt->vcpu)) {
1638 kvm_inject_gp(ctxt->vcpu, 0);
1642 /* sysexit must be called from CPL 0 */
1643 if (kvm_x86_ops->get_cpl(ctxt->vcpu) != 0) {
1644 kvm_inject_gp(ctxt->vcpu, 0);
1648 setup_syscalls_segments(ctxt, &cs, &ss);
1650 if ((c->rex_prefix & 0x8) != 0x0)
1651 usermode = X86EMUL_MODE_PROT64;
1653 usermode = X86EMUL_MODE_PROT32;
1657 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1659 case X86EMUL_MODE_PROT32:
1660 cs.selector = (u16)(msr_data + 16);
1661 if ((msr_data & 0xfffc) == 0x0) {
1662 kvm_inject_gp(ctxt->vcpu, 0);
1665 ss.selector = (u16)(msr_data + 24);
1667 case X86EMUL_MODE_PROT64:
1668 cs.selector = (u16)(msr_data + 32);
1669 if (msr_data == 0x0) {
1670 kvm_inject_gp(ctxt->vcpu, 0);
1673 ss.selector = cs.selector + 8;
1678 cs.selector |= SELECTOR_RPL_MASK;
1679 ss.selector |= SELECTOR_RPL_MASK;
1681 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1682 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1684 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1685 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1691 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1693 unsigned long memop = 0;
1695 unsigned long saved_eip = 0;
1696 struct decode_cache *c = &ctxt->decode;
1701 ctxt->interruptibility = 0;
1703 /* Shadow copy of register state. Committed on successful emulation.
1704 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1708 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1711 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1712 memop = c->modrm_ea;
1714 if (c->rep_prefix && (c->d & String)) {
1715 /* All REP prefixes have the same first termination condition */
1716 if (c->regs[VCPU_REGS_RCX] == 0) {
1717 kvm_rip_write(ctxt->vcpu, c->eip);
1720 /* The second termination condition only applies for REPE
1721 * and REPNE. Test if the repeat string operation prefix is
1722 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1723 * corresponding termination condition according to:
1724 * - if REPE/REPZ and ZF = 0 then done
1725 * - if REPNE/REPNZ and ZF = 1 then done
1727 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1728 (c->b == 0xae) || (c->b == 0xaf)) {
1729 if ((c->rep_prefix == REPE_PREFIX) &&
1730 ((ctxt->eflags & EFLG_ZF) == 0)) {
1731 kvm_rip_write(ctxt->vcpu, c->eip);
1734 if ((c->rep_prefix == REPNE_PREFIX) &&
1735 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1736 kvm_rip_write(ctxt->vcpu, c->eip);
1740 c->regs[VCPU_REGS_RCX]--;
1741 c->eip = kvm_rip_read(ctxt->vcpu);
1744 if (c->src.type == OP_MEM) {
1745 c->src.ptr = (unsigned long *)memop;
1747 rc = ops->read_emulated((unsigned long)c->src.ptr,
1751 if (rc != X86EMUL_CONTINUE)
1753 c->src.orig_val = c->src.val;
1756 if ((c->d & DstMask) == ImplicitOps)
1760 if (c->dst.type == OP_MEM) {
1761 c->dst.ptr = (unsigned long *)memop;
1762 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1765 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1767 c->dst.ptr = (void *)c->dst.ptr +
1768 (c->src.val & mask) / 8;
1770 if (!(c->d & Mov)) {
1771 /* optimisation - avoid slow emulated read */
1772 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1776 if (rc != X86EMUL_CONTINUE)
1780 c->dst.orig_val = c->dst.val;
1790 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1792 case 0x06: /* push es */
1793 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1795 case 0x07: /* pop es */
1796 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1802 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1804 case 0x0e: /* push cs */
1805 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1809 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1811 case 0x16: /* push ss */
1812 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1814 case 0x17: /* pop ss */
1815 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1821 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1823 case 0x1e: /* push ds */
1824 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1826 case 0x1f: /* pop ds */
1827 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1833 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1837 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1841 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1845 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1847 case 0x40 ... 0x47: /* inc r16/r32 */
1848 emulate_1op("inc", c->dst, ctxt->eflags);
1850 case 0x48 ... 0x4f: /* dec r16/r32 */
1851 emulate_1op("dec", c->dst, ctxt->eflags);
1853 case 0x50 ... 0x57: /* push reg */
1856 case 0x58 ... 0x5f: /* pop reg */
1858 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1862 case 0x60: /* pusha */
1863 emulate_pusha(ctxt);
1865 case 0x61: /* popa */
1866 rc = emulate_popa(ctxt, ops);
1870 case 0x63: /* movsxd */
1871 if (ctxt->mode != X86EMUL_MODE_PROT64)
1872 goto cannot_emulate;
1873 c->dst.val = (s32) c->src.val;
1875 case 0x68: /* push imm */
1876 case 0x6a: /* push imm8 */
1879 case 0x6c: /* insb */
1880 case 0x6d: /* insw/insd */
1881 if (kvm_emulate_pio_string(ctxt->vcpu,
1883 (c->d & ByteOp) ? 1 : c->op_bytes,
1885 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1886 (ctxt->eflags & EFLG_DF),
1887 register_address(c, es_base(ctxt),
1888 c->regs[VCPU_REGS_RDI]),
1890 c->regs[VCPU_REGS_RDX]) == 0) {
1895 case 0x6e: /* outsb */
1896 case 0x6f: /* outsw/outsd */
1897 if (kvm_emulate_pio_string(ctxt->vcpu,
1899 (c->d & ByteOp) ? 1 : c->op_bytes,
1901 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1902 (ctxt->eflags & EFLG_DF),
1904 seg_override_base(ctxt, c),
1905 c->regs[VCPU_REGS_RSI]),
1907 c->regs[VCPU_REGS_RDX]) == 0) {
1912 case 0x70 ... 0x7f: /* jcc (short) */
1913 if (test_cc(c->b, ctxt->eflags))
1914 jmp_rel(c, c->src.val);
1916 case 0x80 ... 0x83: /* Grp1 */
1917 switch (c->modrm_reg) {
1937 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1939 case 0x86 ... 0x87: /* xchg */
1941 /* Write back the register source. */
1942 switch (c->dst.bytes) {
1944 *(u8 *) c->src.ptr = (u8) c->dst.val;
1947 *(u16 *) c->src.ptr = (u16) c->dst.val;
1950 *c->src.ptr = (u32) c->dst.val;
1951 break; /* 64b reg: zero-extend */
1953 *c->src.ptr = c->dst.val;
1957 * Write back the memory destination with implicit LOCK
1960 c->dst.val = c->src.val;
1963 case 0x88 ... 0x8b: /* mov */
1965 case 0x8c: { /* mov r/m, sreg */
1966 struct kvm_segment segreg;
1968 if (c->modrm_reg <= 5)
1969 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
1971 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
1973 goto cannot_emulate;
1975 c->dst.val = segreg.selector;
1978 case 0x8d: /* lea r16/r32, m */
1979 c->dst.val = c->modrm_ea;
1981 case 0x8e: { /* mov seg, r/m16 */
1987 if (c->modrm_reg == VCPU_SREG_SS)
1988 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
1990 if (c->modrm_reg <= 5) {
1991 type_bits = (c->modrm_reg == 1) ? 9 : 1;
1992 err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
1993 type_bits, c->modrm_reg);
1995 printk(KERN_INFO "Invalid segreg in modrm byte 0x%02x\n",
1997 goto cannot_emulate;
2001 goto cannot_emulate;
2003 c->dst.type = OP_NONE; /* Disable writeback. */
2006 case 0x8f: /* pop (sole member of Grp1a) */
2007 rc = emulate_grp1a(ctxt, ops);
2011 case 0x90: /* nop / xchg r8,rax */
2012 if (!(c->rex_prefix & 1)) { /* nop */
2013 c->dst.type = OP_NONE;
2016 case 0x91 ... 0x97: /* xchg reg,rax */
2017 c->src.type = c->dst.type = OP_REG;
2018 c->src.bytes = c->dst.bytes = c->op_bytes;
2019 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2020 c->src.val = *(c->src.ptr);
2022 case 0x9c: /* pushf */
2023 c->src.val = (unsigned long) ctxt->eflags;
2026 case 0x9d: /* popf */
2027 c->dst.type = OP_REG;
2028 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2029 c->dst.bytes = c->op_bytes;
2030 goto pop_instruction;
2031 case 0xa0 ... 0xa1: /* mov */
2032 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2033 c->dst.val = c->src.val;
2035 case 0xa2 ... 0xa3: /* mov */
2036 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2038 case 0xa4 ... 0xa5: /* movs */
2039 c->dst.type = OP_MEM;
2040 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2041 c->dst.ptr = (unsigned long *)register_address(c,
2043 c->regs[VCPU_REGS_RDI]);
2044 rc = ops->read_emulated(register_address(c,
2045 seg_override_base(ctxt, c),
2046 c->regs[VCPU_REGS_RSI]),
2048 c->dst.bytes, ctxt->vcpu);
2049 if (rc != X86EMUL_CONTINUE)
2051 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2052 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2054 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2055 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2058 case 0xa6 ... 0xa7: /* cmps */
2059 c->src.type = OP_NONE; /* Disable writeback. */
2060 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2061 c->src.ptr = (unsigned long *)register_address(c,
2062 seg_override_base(ctxt, c),
2063 c->regs[VCPU_REGS_RSI]);
2064 rc = ops->read_emulated((unsigned long)c->src.ptr,
2068 if (rc != X86EMUL_CONTINUE)
2071 c->dst.type = OP_NONE; /* Disable writeback. */
2072 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2073 c->dst.ptr = (unsigned long *)register_address(c,
2075 c->regs[VCPU_REGS_RDI]);
2076 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2080 if (rc != X86EMUL_CONTINUE)
2083 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2085 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2087 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2088 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2090 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2091 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2095 case 0xaa ... 0xab: /* stos */
2096 c->dst.type = OP_MEM;
2097 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2098 c->dst.ptr = (unsigned long *)register_address(c,
2100 c->regs[VCPU_REGS_RDI]);
2101 c->dst.val = c->regs[VCPU_REGS_RAX];
2102 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2103 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2106 case 0xac ... 0xad: /* lods */
2107 c->dst.type = OP_REG;
2108 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2109 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2110 rc = ops->read_emulated(register_address(c,
2111 seg_override_base(ctxt, c),
2112 c->regs[VCPU_REGS_RSI]),
2116 if (rc != X86EMUL_CONTINUE)
2118 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2119 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2122 case 0xae ... 0xaf: /* scas */
2123 DPRINTF("Urk! I don't handle SCAS.\n");
2124 goto cannot_emulate;
2125 case 0xb0 ... 0xbf: /* mov r, imm */
2130 case 0xc3: /* ret */
2131 c->dst.type = OP_REG;
2132 c->dst.ptr = &c->eip;
2133 c->dst.bytes = c->op_bytes;
2134 goto pop_instruction;
2135 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2137 c->dst.val = c->src.val;
2139 case 0xcb: /* ret far */
2140 rc = emulate_ret_far(ctxt, ops);
2144 case 0xd0 ... 0xd1: /* Grp2 */
2148 case 0xd2 ... 0xd3: /* Grp2 */
2149 c->src.val = c->regs[VCPU_REGS_RCX];
2152 case 0xe4: /* inb */
2157 case 0xe6: /* outb */
2158 case 0xe7: /* out */
2162 case 0xe8: /* call (near) */ {
2163 long int rel = c->src.val;
2164 c->src.val = (unsigned long) c->eip;
2169 case 0xe9: /* jmp rel */
2171 case 0xea: /* jmp far */
2172 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val, 9,
2173 VCPU_SREG_CS) < 0) {
2174 DPRINTF("jmp far: Failed to load CS descriptor\n");
2175 goto cannot_emulate;
2178 c->eip = c->src.val;
2181 jmp: /* jmp rel short */
2182 jmp_rel(c, c->src.val);
2183 c->dst.type = OP_NONE; /* Disable writeback. */
2185 case 0xec: /* in al,dx */
2186 case 0xed: /* in (e/r)ax,dx */
2187 port = c->regs[VCPU_REGS_RDX];
2190 case 0xee: /* out al,dx */
2191 case 0xef: /* out (e/r)ax,dx */
2192 port = c->regs[VCPU_REGS_RDX];
2194 do_io: if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2195 (c->d & ByteOp) ? 1 : c->op_bytes,
2198 goto cannot_emulate;
2201 case 0xf4: /* hlt */
2202 ctxt->vcpu->arch.halt_request = 1;
2204 case 0xf5: /* cmc */
2205 /* complement carry flag from eflags reg */
2206 ctxt->eflags ^= EFLG_CF;
2207 c->dst.type = OP_NONE; /* Disable writeback. */
2209 case 0xf6 ... 0xf7: /* Grp3 */
2210 rc = emulate_grp3(ctxt, ops);
2214 case 0xf8: /* clc */
2215 ctxt->eflags &= ~EFLG_CF;
2216 c->dst.type = OP_NONE; /* Disable writeback. */
2218 case 0xfa: /* cli */
2219 ctxt->eflags &= ~X86_EFLAGS_IF;
2220 c->dst.type = OP_NONE; /* Disable writeback. */
2222 case 0xfb: /* sti */
2223 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2224 ctxt->eflags |= X86_EFLAGS_IF;
2225 c->dst.type = OP_NONE; /* Disable writeback. */
2227 case 0xfc: /* cld */
2228 ctxt->eflags &= ~EFLG_DF;
2229 c->dst.type = OP_NONE; /* Disable writeback. */
2231 case 0xfd: /* std */
2232 ctxt->eflags |= EFLG_DF;
2233 c->dst.type = OP_NONE; /* Disable writeback. */
2235 case 0xfe ... 0xff: /* Grp4/Grp5 */
2236 rc = emulate_grp45(ctxt, ops);
2243 rc = writeback(ctxt, ops);
2247 /* Commit shadow register state. */
2248 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2249 kvm_rip_write(ctxt->vcpu, c->eip);
2252 if (rc == X86EMUL_UNHANDLEABLE) {
2260 case 0x01: /* lgdt, lidt, lmsw */
2261 switch (c->modrm_reg) {
2263 unsigned long address;
2265 case 0: /* vmcall */
2266 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2267 goto cannot_emulate;
2269 rc = kvm_fix_hypercall(ctxt->vcpu);
2273 /* Let the processor re-execute the fixed hypercall */
2274 c->eip = kvm_rip_read(ctxt->vcpu);
2275 /* Disable writeback. */
2276 c->dst.type = OP_NONE;
2279 rc = read_descriptor(ctxt, ops, c->src.ptr,
2280 &size, &address, c->op_bytes);
2283 realmode_lgdt(ctxt->vcpu, size, address);
2284 /* Disable writeback. */
2285 c->dst.type = OP_NONE;
2287 case 3: /* lidt/vmmcall */
2288 if (c->modrm_mod == 3) {
2289 switch (c->modrm_rm) {
2291 rc = kvm_fix_hypercall(ctxt->vcpu);
2296 goto cannot_emulate;
2299 rc = read_descriptor(ctxt, ops, c->src.ptr,
2304 realmode_lidt(ctxt->vcpu, size, address);
2306 /* Disable writeback. */
2307 c->dst.type = OP_NONE;
2311 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2314 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2316 c->dst.type = OP_NONE;
2319 emulate_invlpg(ctxt->vcpu, memop);
2320 /* Disable writeback. */
2321 c->dst.type = OP_NONE;
2324 goto cannot_emulate;
2327 case 0x05: /* syscall */
2328 if (emulate_syscall(ctxt) == -1)
2329 goto cannot_emulate;
2334 emulate_clts(ctxt->vcpu);
2335 c->dst.type = OP_NONE;
2337 case 0x08: /* invd */
2338 case 0x09: /* wbinvd */
2339 case 0x0d: /* GrpP (prefetch) */
2340 case 0x18: /* Grp16 (prefetch/nop) */
2341 c->dst.type = OP_NONE;
2343 case 0x20: /* mov cr, reg */
2344 if (c->modrm_mod != 3)
2345 goto cannot_emulate;
2346 c->regs[c->modrm_rm] =
2347 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2348 c->dst.type = OP_NONE; /* no writeback */
2350 case 0x21: /* mov from dr to reg */
2351 if (c->modrm_mod != 3)
2352 goto cannot_emulate;
2353 rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
2355 goto cannot_emulate;
2356 c->dst.type = OP_NONE; /* no writeback */
2358 case 0x22: /* mov reg, cr */
2359 if (c->modrm_mod != 3)
2360 goto cannot_emulate;
2361 realmode_set_cr(ctxt->vcpu,
2362 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2363 c->dst.type = OP_NONE;
2365 case 0x23: /* mov from reg to dr */
2366 if (c->modrm_mod != 3)
2367 goto cannot_emulate;
2368 rc = emulator_set_dr(ctxt, c->modrm_reg,
2369 c->regs[c->modrm_rm]);
2371 goto cannot_emulate;
2372 c->dst.type = OP_NONE; /* no writeback */
2376 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2377 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2378 rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
2380 kvm_inject_gp(ctxt->vcpu, 0);
2381 c->eip = kvm_rip_read(ctxt->vcpu);
2383 rc = X86EMUL_CONTINUE;
2384 c->dst.type = OP_NONE;
2388 rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
2390 kvm_inject_gp(ctxt->vcpu, 0);
2391 c->eip = kvm_rip_read(ctxt->vcpu);
2393 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2394 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2396 rc = X86EMUL_CONTINUE;
2397 c->dst.type = OP_NONE;
2399 case 0x34: /* sysenter */
2400 if (emulate_sysenter(ctxt) == -1)
2401 goto cannot_emulate;
2405 case 0x35: /* sysexit */
2406 if (emulate_sysexit(ctxt) == -1)
2407 goto cannot_emulate;
2411 case 0x40 ... 0x4f: /* cmov */
2412 c->dst.val = c->dst.orig_val = c->src.val;
2413 if (!test_cc(c->b, ctxt->eflags))
2414 c->dst.type = OP_NONE; /* no writeback */
2416 case 0x80 ... 0x8f: /* jnz rel, etc*/
2417 if (test_cc(c->b, ctxt->eflags))
2418 jmp_rel(c, c->src.val);
2419 c->dst.type = OP_NONE;
2421 case 0xa0: /* push fs */
2422 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2424 case 0xa1: /* pop fs */
2425 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2431 c->dst.type = OP_NONE;
2432 /* only subword offset */
2433 c->src.val &= (c->dst.bytes << 3) - 1;
2434 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2436 case 0xa4: /* shld imm8, r, r/m */
2437 case 0xa5: /* shld cl, r, r/m */
2438 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2440 case 0xa8: /* push gs */
2441 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2443 case 0xa9: /* pop gs */
2444 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2450 /* only subword offset */
2451 c->src.val &= (c->dst.bytes << 3) - 1;
2452 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2454 case 0xac: /* shrd imm8, r, r/m */
2455 case 0xad: /* shrd cl, r, r/m */
2456 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2458 case 0xae: /* clflush */
2460 case 0xb0 ... 0xb1: /* cmpxchg */
2462 * Save real source value, then compare EAX against
2465 c->src.orig_val = c->src.val;
2466 c->src.val = c->regs[VCPU_REGS_RAX];
2467 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2468 if (ctxt->eflags & EFLG_ZF) {
2469 /* Success: write back to memory. */
2470 c->dst.val = c->src.orig_val;
2472 /* Failure: write the value we saw to EAX. */
2473 c->dst.type = OP_REG;
2474 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2479 /* only subword offset */
2480 c->src.val &= (c->dst.bytes << 3) - 1;
2481 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2483 case 0xb6 ... 0xb7: /* movzx */
2484 c->dst.bytes = c->op_bytes;
2485 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2488 case 0xba: /* Grp8 */
2489 switch (c->modrm_reg & 3) {
2502 /* only subword offset */
2503 c->src.val &= (c->dst.bytes << 3) - 1;
2504 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2506 case 0xbe ... 0xbf: /* movsx */
2507 c->dst.bytes = c->op_bytes;
2508 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2511 case 0xc3: /* movnti */
2512 c->dst.bytes = c->op_bytes;
2513 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2516 case 0xc7: /* Grp9 (cmpxchg8b) */
2517 rc = emulate_grp9(ctxt, ops, memop);
2520 c->dst.type = OP_NONE;
2526 DPRINTF("Cannot emulate %02x\n", c->b);