1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
80 /* Source 2 operand type */
81 #define Src2None (0<<29)
82 #define Src2CL (1<<29)
83 #define Src2ImmByte (2<<29)
84 #define Src2One (3<<29)
85 #define Src2Imm16 (4<<29)
86 #define Src2Mask (7<<29)
89 Group1_80, Group1_81, Group1_82, Group1_83,
90 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
94 static u32 opcode_table[256] = {
96 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
97 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
98 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
99 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
101 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
102 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
103 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
104 ImplicitOps | Stack | No64, 0,
106 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
107 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
108 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
109 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
111 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
112 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
113 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
114 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
116 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
117 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
118 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
120 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
121 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
124 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
125 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
128 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
129 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
133 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
135 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
137 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
138 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
140 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
141 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
143 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
144 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
147 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
148 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
149 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
151 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
152 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
155 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 Group | Group1_80, Group | Group1_81,
158 Group | Group1_82, Group | Group1_83,
159 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
160 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
162 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
163 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
164 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
165 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
167 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
169 0, 0, SrcImm | Src2Imm16 | No64, 0,
170 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
172 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
173 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
174 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
175 ByteOp | ImplicitOps | String, ImplicitOps | String,
177 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
178 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
179 ByteOp | ImplicitOps | String, ImplicitOps | String,
181 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
182 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
184 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
186 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
187 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
189 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
191 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
192 0, ImplicitOps | Stack, 0, 0,
193 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
195 0, 0, 0, ImplicitOps | Stack,
196 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
198 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
199 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
202 0, 0, 0, 0, 0, 0, 0, 0,
205 ByteOp | SrcImmUByte, SrcImmUByte,
206 ByteOp | SrcImmUByte, SrcImmUByte,
208 SrcImm | Stack, SrcImm | ImplicitOps,
209 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
210 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
211 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
214 ImplicitOps, ImplicitOps, Group | Group3_Byte, Group | Group3,
216 ImplicitOps, 0, ImplicitOps, ImplicitOps,
217 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
220 static u32 twobyte_table[256] = {
222 0, Group | GroupDual | Group7, 0, 0, 0, ImplicitOps, ImplicitOps, 0,
223 ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0,
225 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
227 ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0,
228 0, 0, 0, 0, 0, 0, 0, 0,
230 ImplicitOps, 0, ImplicitOps, 0,
231 ImplicitOps, ImplicitOps, 0, 0,
232 0, 0, 0, 0, 0, 0, 0, 0,
234 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
235 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
236 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
237 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
239 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
244 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
246 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
248 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
250 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
251 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
253 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
255 ImplicitOps | Stack, ImplicitOps | Stack,
256 0, DstMem | SrcReg | ModRM | BitOp,
257 DstMem | SrcReg | Src2ImmByte | ModRM,
258 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
260 ImplicitOps | Stack, ImplicitOps | Stack,
261 0, DstMem | SrcReg | ModRM | BitOp,
262 DstMem | SrcReg | Src2ImmByte | ModRM,
263 DstMem | SrcReg | Src2CL | ModRM,
266 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0,
267 DstMem | SrcReg | ModRM | BitOp,
268 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
269 DstReg | SrcMem16 | ModRM | Mov,
271 0, 0, Group | Group8, DstMem | SrcReg | ModRM | BitOp,
272 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
273 DstReg | SrcMem16 | ModRM | Mov,
275 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
276 0, 0, 0, Group | GroupDual | Group9,
277 0, 0, 0, 0, 0, 0, 0, 0,
279 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
281 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
283 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
286 static u32 group_table[] = {
288 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
289 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
290 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
291 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
293 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
294 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
295 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
296 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
298 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
299 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
300 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
301 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
303 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
304 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
305 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
306 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
308 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
310 ByteOp | SrcImm | DstMem | ModRM, 0,
311 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
314 DstMem | SrcImm | ModRM, 0,
315 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
318 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
321 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
322 SrcMem | ModRM | Stack, 0,
323 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
325 0, 0, ModRM | SrcMem, ModRM | SrcMem,
326 SrcNone | ModRM | DstMem | Mov, 0,
327 SrcMem16 | ModRM | Mov, SrcMem | ModRM | ByteOp,
330 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
331 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
333 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0,
336 static u32 group2_table[] = {
338 SrcNone | ModRM, 0, 0, SrcNone | ModRM,
339 SrcNone | ModRM | DstMem | Mov, 0,
340 SrcMem16 | ModRM | Mov, 0,
342 0, 0, 0, 0, 0, 0, 0, 0,
345 /* EFLAGS bit definitions. */
346 #define EFLG_VM (1<<17)
347 #define EFLG_RF (1<<16)
348 #define EFLG_OF (1<<11)
349 #define EFLG_DF (1<<10)
350 #define EFLG_IF (1<<9)
351 #define EFLG_SF (1<<7)
352 #define EFLG_ZF (1<<6)
353 #define EFLG_AF (1<<4)
354 #define EFLG_PF (1<<2)
355 #define EFLG_CF (1<<0)
358 * Instruction emulation:
359 * Most instructions are emulated directly via a fragment of inline assembly
360 * code. This allows us to save/restore EFLAGS and thus very easily pick up
361 * any modified flags.
364 #if defined(CONFIG_X86_64)
365 #define _LO32 "k" /* force 32-bit operand */
366 #define _STK "%%rsp" /* stack pointer */
367 #elif defined(__i386__)
368 #define _LO32 "" /* force 32-bit operand */
369 #define _STK "%%esp" /* stack pointer */
373 * These EFLAGS bits are restored from saved value during emulation, and
374 * any changes are written back to the saved value after emulation.
376 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
378 /* Before executing instruction: restore necessary bits in EFLAGS. */
379 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
380 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
381 "movl %"_sav",%"_LO32 _tmp"; " \
384 "movl %"_msk",%"_LO32 _tmp"; " \
385 "andl %"_LO32 _tmp",("_STK"); " \
387 "notl %"_LO32 _tmp"; " \
388 "andl %"_LO32 _tmp",("_STK"); " \
389 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
391 "orl %"_LO32 _tmp",("_STK"); " \
395 /* After executing instruction: write-back necessary bits in EFLAGS. */
396 #define _POST_EFLAGS(_sav, _msk, _tmp) \
397 /* _sav |= EFLAGS & _msk; */ \
400 "andl %"_msk",%"_LO32 _tmp"; " \
401 "orl %"_LO32 _tmp",%"_sav"; "
409 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
411 __asm__ __volatile__ ( \
412 _PRE_EFLAGS("0", "4", "2") \
413 _op _suffix " %"_x"3,%1; " \
414 _POST_EFLAGS("0", "4", "2") \
415 : "=m" (_eflags), "=m" ((_dst).val), \
417 : _y ((_src).val), "i" (EFLAGS_MASK)); \
421 /* Raw emulation: instruction has two explicit operands. */
422 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
424 unsigned long _tmp; \
426 switch ((_dst).bytes) { \
428 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
431 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
434 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
439 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
441 unsigned long _tmp; \
442 switch ((_dst).bytes) { \
444 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
447 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
448 _wx, _wy, _lx, _ly, _qx, _qy); \
453 /* Source operand is byte-sized and may be restricted to just %cl. */
454 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
455 __emulate_2op(_op, _src, _dst, _eflags, \
456 "b", "c", "b", "c", "b", "c", "b", "c")
458 /* Source operand is byte, word, long or quad sized. */
459 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
460 __emulate_2op(_op, _src, _dst, _eflags, \
461 "b", "q", "w", "r", _LO32, "r", "", "r")
463 /* Source operand is word, long or quad sized. */
464 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
465 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
466 "w", "r", _LO32, "r", "", "r")
468 /* Instruction has three operands and one operand is stored in ECX register */
469 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
471 unsigned long _tmp; \
472 _type _clv = (_cl).val; \
473 _type _srcv = (_src).val; \
474 _type _dstv = (_dst).val; \
476 __asm__ __volatile__ ( \
477 _PRE_EFLAGS("0", "5", "2") \
478 _op _suffix " %4,%1 \n" \
479 _POST_EFLAGS("0", "5", "2") \
480 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
481 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
484 (_cl).val = (unsigned long) _clv; \
485 (_src).val = (unsigned long) _srcv; \
486 (_dst).val = (unsigned long) _dstv; \
489 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
491 switch ((_dst).bytes) { \
493 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
494 "w", unsigned short); \
497 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
498 "l", unsigned int); \
501 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
502 "q", unsigned long)); \
507 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
509 unsigned long _tmp; \
511 __asm__ __volatile__ ( \
512 _PRE_EFLAGS("0", "3", "2") \
513 _op _suffix " %1; " \
514 _POST_EFLAGS("0", "3", "2") \
515 : "=m" (_eflags), "+m" ((_dst).val), \
517 : "i" (EFLAGS_MASK)); \
520 /* Instruction has only one explicit operand (no source operand). */
521 #define emulate_1op(_op, _dst, _eflags) \
523 switch ((_dst).bytes) { \
524 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
525 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
526 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
527 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
531 /* Fetch next part of the instruction being emulated. */
532 #define insn_fetch(_type, _size, _eip) \
533 ({ unsigned long _x; \
534 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
541 static inline unsigned long ad_mask(struct decode_cache *c)
543 return (1UL << (c->ad_bytes << 3)) - 1;
546 /* Access/update address held in a register, based on addressing mode. */
547 static inline unsigned long
548 address_mask(struct decode_cache *c, unsigned long reg)
550 if (c->ad_bytes == sizeof(unsigned long))
553 return reg & ad_mask(c);
556 static inline unsigned long
557 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
559 return base + address_mask(c, reg);
563 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
565 if (c->ad_bytes == sizeof(unsigned long))
568 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
571 static inline void jmp_rel(struct decode_cache *c, int rel)
573 register_address_increment(c, &c->eip, rel);
576 static void set_seg_override(struct decode_cache *c, int seg)
578 c->has_seg_override = true;
579 c->seg_override = seg;
582 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
584 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
587 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
590 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
591 struct decode_cache *c)
593 if (!c->has_seg_override)
596 return seg_base(ctxt, c->seg_override);
599 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
601 return seg_base(ctxt, VCPU_SREG_ES);
604 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
606 return seg_base(ctxt, VCPU_SREG_SS);
609 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
610 struct x86_emulate_ops *ops,
611 unsigned long linear, u8 *dest)
613 struct fetch_cache *fc = &ctxt->decode.fetch;
617 if (linear < fc->start || linear >= fc->end) {
618 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
619 rc = ops->fetch(linear, fc->data, size, ctxt->vcpu, NULL);
623 fc->end = linear + size;
625 *dest = fc->data[linear - fc->start];
629 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
630 struct x86_emulate_ops *ops,
631 unsigned long eip, void *dest, unsigned size)
635 /* x86 instructions are limited to 15 bytes. */
636 if (eip + size - ctxt->decode.eip_orig > 15)
637 return X86EMUL_UNHANDLEABLE;
638 eip += ctxt->cs_base;
640 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
648 * Given the 'reg' portion of a ModRM byte, and a register block, return a
649 * pointer into the block that addresses the relevant register.
650 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
652 static void *decode_register(u8 modrm_reg, unsigned long *regs,
657 p = ®s[modrm_reg];
658 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
659 p = (unsigned char *)®s[modrm_reg & 3] + 1;
663 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
664 struct x86_emulate_ops *ops,
666 u16 *size, unsigned long *address, int op_bytes)
673 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
677 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
682 static int test_cc(unsigned int condition, unsigned int flags)
686 switch ((condition & 15) >> 1) {
688 rc |= (flags & EFLG_OF);
690 case 1: /* b/c/nae */
691 rc |= (flags & EFLG_CF);
694 rc |= (flags & EFLG_ZF);
697 rc |= (flags & (EFLG_CF|EFLG_ZF));
700 rc |= (flags & EFLG_SF);
703 rc |= (flags & EFLG_PF);
706 rc |= (flags & EFLG_ZF);
709 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
713 /* Odd condition identifiers (lsb == 1) have inverted sense. */
714 return (!!rc ^ (condition & 1));
717 static void decode_register_operand(struct operand *op,
718 struct decode_cache *c,
721 unsigned reg = c->modrm_reg;
722 int highbyte_regs = c->rex_prefix == 0;
725 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
727 if ((c->d & ByteOp) && !inhibit_bytereg) {
728 op->ptr = decode_register(reg, c->regs, highbyte_regs);
729 op->val = *(u8 *)op->ptr;
732 op->ptr = decode_register(reg, c->regs, 0);
733 op->bytes = c->op_bytes;
736 op->val = *(u16 *)op->ptr;
739 op->val = *(u32 *)op->ptr;
742 op->val = *(u64 *) op->ptr;
746 op->orig_val = op->val;
749 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
750 struct x86_emulate_ops *ops)
752 struct decode_cache *c = &ctxt->decode;
754 int index_reg = 0, base_reg = 0, scale;
758 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
759 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
760 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
763 c->modrm = insn_fetch(u8, 1, c->eip);
764 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
765 c->modrm_reg |= (c->modrm & 0x38) >> 3;
766 c->modrm_rm |= (c->modrm & 0x07);
770 if (c->modrm_mod == 3) {
771 c->modrm_ptr = decode_register(c->modrm_rm,
772 c->regs, c->d & ByteOp);
773 c->modrm_val = *(unsigned long *)c->modrm_ptr;
777 if (c->ad_bytes == 2) {
778 unsigned bx = c->regs[VCPU_REGS_RBX];
779 unsigned bp = c->regs[VCPU_REGS_RBP];
780 unsigned si = c->regs[VCPU_REGS_RSI];
781 unsigned di = c->regs[VCPU_REGS_RDI];
783 /* 16-bit ModR/M decode. */
784 switch (c->modrm_mod) {
786 if (c->modrm_rm == 6)
787 c->modrm_ea += insn_fetch(u16, 2, c->eip);
790 c->modrm_ea += insn_fetch(s8, 1, c->eip);
793 c->modrm_ea += insn_fetch(u16, 2, c->eip);
796 switch (c->modrm_rm) {
798 c->modrm_ea += bx + si;
801 c->modrm_ea += bx + di;
804 c->modrm_ea += bp + si;
807 c->modrm_ea += bp + di;
816 if (c->modrm_mod != 0)
823 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
824 (c->modrm_rm == 6 && c->modrm_mod != 0))
825 if (!c->has_seg_override)
826 set_seg_override(c, VCPU_SREG_SS);
827 c->modrm_ea = (u16)c->modrm_ea;
829 /* 32/64-bit ModR/M decode. */
830 if ((c->modrm_rm & 7) == 4) {
831 sib = insn_fetch(u8, 1, c->eip);
832 index_reg |= (sib >> 3) & 7;
836 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
837 c->modrm_ea += insn_fetch(s32, 4, c->eip);
839 c->modrm_ea += c->regs[base_reg];
841 c->modrm_ea += c->regs[index_reg] << scale;
842 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
843 if (ctxt->mode == X86EMUL_MODE_PROT64)
846 c->modrm_ea += c->regs[c->modrm_rm];
847 switch (c->modrm_mod) {
849 if (c->modrm_rm == 5)
850 c->modrm_ea += insn_fetch(s32, 4, c->eip);
853 c->modrm_ea += insn_fetch(s8, 1, c->eip);
856 c->modrm_ea += insn_fetch(s32, 4, c->eip);
864 static int decode_abs(struct x86_emulate_ctxt *ctxt,
865 struct x86_emulate_ops *ops)
867 struct decode_cache *c = &ctxt->decode;
870 switch (c->ad_bytes) {
872 c->modrm_ea = insn_fetch(u16, 2, c->eip);
875 c->modrm_ea = insn_fetch(u32, 4, c->eip);
878 c->modrm_ea = insn_fetch(u64, 8, c->eip);
886 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
888 struct decode_cache *c = &ctxt->decode;
890 int mode = ctxt->mode;
891 int def_op_bytes, def_ad_bytes, group;
893 /* Shadow copy of register state. Committed on successful emulation. */
895 memset(c, 0, sizeof(struct decode_cache));
896 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
897 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
898 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
901 case X86EMUL_MODE_REAL:
902 case X86EMUL_MODE_VM86:
903 case X86EMUL_MODE_PROT16:
904 def_op_bytes = def_ad_bytes = 2;
906 case X86EMUL_MODE_PROT32:
907 def_op_bytes = def_ad_bytes = 4;
910 case X86EMUL_MODE_PROT64:
919 c->op_bytes = def_op_bytes;
920 c->ad_bytes = def_ad_bytes;
922 /* Legacy prefixes. */
924 switch (c->b = insn_fetch(u8, 1, c->eip)) {
925 case 0x66: /* operand-size override */
926 /* switch between 2/4 bytes */
927 c->op_bytes = def_op_bytes ^ 6;
929 case 0x67: /* address-size override */
930 if (mode == X86EMUL_MODE_PROT64)
931 /* switch between 4/8 bytes */
932 c->ad_bytes = def_ad_bytes ^ 12;
934 /* switch between 2/4 bytes */
935 c->ad_bytes = def_ad_bytes ^ 6;
937 case 0x26: /* ES override */
938 case 0x2e: /* CS override */
939 case 0x36: /* SS override */
940 case 0x3e: /* DS override */
941 set_seg_override(c, (c->b >> 3) & 3);
943 case 0x64: /* FS override */
944 case 0x65: /* GS override */
945 set_seg_override(c, c->b & 7);
947 case 0x40 ... 0x4f: /* REX */
948 if (mode != X86EMUL_MODE_PROT64)
950 c->rex_prefix = c->b;
952 case 0xf0: /* LOCK */
955 case 0xf2: /* REPNE/REPNZ */
956 c->rep_prefix = REPNE_PREFIX;
958 case 0xf3: /* REP/REPE/REPZ */
959 c->rep_prefix = REPE_PREFIX;
965 /* Any legacy prefix after a REX prefix nullifies its effect. */
974 if (c->rex_prefix & 8)
975 c->op_bytes = 8; /* REX.W */
977 /* Opcode byte(s). */
978 c->d = opcode_table[c->b];
980 /* Two-byte opcode? */
983 c->b = insn_fetch(u8, 1, c->eip);
984 c->d = twobyte_table[c->b];
988 if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
989 kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");;
994 group = c->d & GroupMask;
995 c->modrm = insn_fetch(u8, 1, c->eip);
998 group = (group << 3) + ((c->modrm >> 3) & 7);
999 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1000 c->d = group2_table[group];
1002 c->d = group_table[group];
1007 DPRINTF("Cannot emulate %02x\n", c->b);
1011 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1014 /* ModRM and SIB bytes. */
1016 rc = decode_modrm(ctxt, ops);
1017 else if (c->d & MemAbs)
1018 rc = decode_abs(ctxt, ops);
1022 if (!c->has_seg_override)
1023 set_seg_override(c, VCPU_SREG_DS);
1025 if (!(!c->twobyte && c->b == 0x8d))
1026 c->modrm_ea += seg_override_base(ctxt, c);
1028 if (c->ad_bytes != 8)
1029 c->modrm_ea = (u32)c->modrm_ea;
1031 * Decode and fetch the source operand: register, memory
1034 switch (c->d & SrcMask) {
1038 decode_register_operand(&c->src, c, 0);
1047 c->src.bytes = (c->d & ByteOp) ? 1 :
1049 /* Don't fetch the address for invlpg: it could be unmapped. */
1050 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1054 * For instructions with a ModR/M byte, switch to register
1055 * access if Mod = 3.
1057 if ((c->d & ModRM) && c->modrm_mod == 3) {
1058 c->src.type = OP_REG;
1059 c->src.val = c->modrm_val;
1060 c->src.ptr = c->modrm_ptr;
1063 c->src.type = OP_MEM;
1067 c->src.type = OP_IMM;
1068 c->src.ptr = (unsigned long *)c->eip;
1069 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1070 if (c->src.bytes == 8)
1072 /* NB. Immediates are sign-extended as necessary. */
1073 switch (c->src.bytes) {
1075 c->src.val = insn_fetch(s8, 1, c->eip);
1078 c->src.val = insn_fetch(s16, 2, c->eip);
1081 c->src.val = insn_fetch(s32, 4, c->eip);
1084 if ((c->d & SrcMask) == SrcImmU) {
1085 switch (c->src.bytes) {
1090 c->src.val &= 0xffff;
1093 c->src.val &= 0xffffffff;
1100 c->src.type = OP_IMM;
1101 c->src.ptr = (unsigned long *)c->eip;
1103 if ((c->d & SrcMask) == SrcImmByte)
1104 c->src.val = insn_fetch(s8, 1, c->eip);
1106 c->src.val = insn_fetch(u8, 1, c->eip);
1115 * Decode and fetch the second source operand: register, memory
1118 switch (c->d & Src2Mask) {
1123 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1126 c->src2.type = OP_IMM;
1127 c->src2.ptr = (unsigned long *)c->eip;
1129 c->src2.val = insn_fetch(u8, 1, c->eip);
1132 c->src2.type = OP_IMM;
1133 c->src2.ptr = (unsigned long *)c->eip;
1135 c->src2.val = insn_fetch(u16, 2, c->eip);
1143 /* Decode and fetch the destination operand: register or memory. */
1144 switch (c->d & DstMask) {
1146 /* Special instructions do their own operand decoding. */
1149 decode_register_operand(&c->dst, c,
1150 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1153 if ((c->d & ModRM) && c->modrm_mod == 3) {
1154 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1155 c->dst.type = OP_REG;
1156 c->dst.val = c->dst.orig_val = c->modrm_val;
1157 c->dst.ptr = c->modrm_ptr;
1160 c->dst.type = OP_MEM;
1163 c->dst.type = OP_REG;
1164 c->dst.bytes = c->op_bytes;
1165 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1166 switch (c->op_bytes) {
1168 c->dst.val = *(u8 *)c->dst.ptr;
1171 c->dst.val = *(u16 *)c->dst.ptr;
1174 c->dst.val = *(u32 *)c->dst.ptr;
1177 c->dst.orig_val = c->dst.val;
1181 if (c->rip_relative)
1182 c->modrm_ea += c->eip;
1185 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1188 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1190 struct decode_cache *c = &ctxt->decode;
1192 c->dst.type = OP_MEM;
1193 c->dst.bytes = c->op_bytes;
1194 c->dst.val = c->src.val;
1195 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1196 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1197 c->regs[VCPU_REGS_RSP]);
1200 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1201 struct x86_emulate_ops *ops,
1202 void *dest, int len)
1204 struct decode_cache *c = &ctxt->decode;
1207 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1208 c->regs[VCPU_REGS_RSP]),
1209 dest, len, ctxt->vcpu);
1210 if (rc != X86EMUL_CONTINUE)
1213 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1217 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1219 struct decode_cache *c = &ctxt->decode;
1220 struct kvm_segment segment;
1222 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1224 c->src.val = segment.selector;
1228 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1229 struct x86_emulate_ops *ops, int seg)
1231 struct decode_cache *c = &ctxt->decode;
1232 unsigned long selector;
1235 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1239 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, 1, seg);
1243 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1245 struct decode_cache *c = &ctxt->decode;
1246 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1247 int reg = VCPU_REGS_RAX;
1249 while (reg <= VCPU_REGS_RDI) {
1250 (reg == VCPU_REGS_RSP) ?
1251 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1258 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1259 struct x86_emulate_ops *ops)
1261 struct decode_cache *c = &ctxt->decode;
1263 int reg = VCPU_REGS_RDI;
1265 while (reg >= VCPU_REGS_RAX) {
1266 if (reg == VCPU_REGS_RSP) {
1267 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1272 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1280 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1281 struct x86_emulate_ops *ops)
1283 struct decode_cache *c = &ctxt->decode;
1286 rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1292 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1294 struct decode_cache *c = &ctxt->decode;
1295 switch (c->modrm_reg) {
1297 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1300 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1303 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1306 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1308 case 4: /* sal/shl */
1309 case 6: /* sal/shl */
1310 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1313 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1316 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1321 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1322 struct x86_emulate_ops *ops)
1324 struct decode_cache *c = &ctxt->decode;
1327 switch (c->modrm_reg) {
1328 case 0 ... 1: /* test */
1329 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1332 c->dst.val = ~c->dst.val;
1335 emulate_1op("neg", c->dst, ctxt->eflags);
1338 DPRINTF("Cannot emulate %02x\n", c->b);
1339 rc = X86EMUL_UNHANDLEABLE;
1345 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1346 struct x86_emulate_ops *ops)
1348 struct decode_cache *c = &ctxt->decode;
1350 switch (c->modrm_reg) {
1352 emulate_1op("inc", c->dst, ctxt->eflags);
1355 emulate_1op("dec", c->dst, ctxt->eflags);
1357 case 2: /* call near abs */ {
1360 c->eip = c->src.val;
1361 c->src.val = old_eip;
1365 case 4: /* jmp abs */
1366 c->eip = c->src.val;
1375 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1376 struct x86_emulate_ops *ops,
1377 unsigned long memop)
1379 struct decode_cache *c = &ctxt->decode;
1383 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1384 if (rc != X86EMUL_CONTINUE)
1387 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1388 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1390 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1391 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1392 ctxt->eflags &= ~EFLG_ZF;
1395 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1396 (u32) c->regs[VCPU_REGS_RBX];
1398 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1399 if (rc != X86EMUL_CONTINUE)
1401 ctxt->eflags |= EFLG_ZF;
1406 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1407 struct x86_emulate_ops *ops)
1409 struct decode_cache *c = &ctxt->decode;
1413 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1416 if (c->op_bytes == 4)
1417 c->eip = (u32)c->eip;
1418 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1421 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, 1, VCPU_SREG_CS);
1425 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1426 struct x86_emulate_ops *ops)
1429 struct decode_cache *c = &ctxt->decode;
1431 switch (c->dst.type) {
1433 /* The 4-byte case *is* correct:
1434 * in 64-bit mode we zero-extend.
1436 switch (c->dst.bytes) {
1438 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1441 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1444 *c->dst.ptr = (u32)c->dst.val;
1445 break; /* 64b: zero-ext */
1447 *c->dst.ptr = c->dst.val;
1453 rc = ops->cmpxchg_emulated(
1454 (unsigned long)c->dst.ptr,
1460 rc = ops->write_emulated(
1461 (unsigned long)c->dst.ptr,
1465 if (rc != X86EMUL_CONTINUE)
1477 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1479 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1481 * an sti; sti; sequence only disable interrupts for the first
1482 * instruction. So, if the last instruction, be it emulated or
1483 * not, left the system with the INT_STI flag enabled, it
1484 * means that the last instruction is an sti. We should not
1485 * leave the flag on in this case. The same goes for mov ss
1487 if (!(int_shadow & mask))
1488 ctxt->interruptibility = mask;
1492 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1493 struct kvm_segment *cs, struct kvm_segment *ss)
1495 memset(cs, 0, sizeof(struct kvm_segment));
1496 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1497 memset(ss, 0, sizeof(struct kvm_segment));
1499 cs->l = 0; /* will be adjusted later */
1500 cs->base = 0; /* flat segment */
1501 cs->g = 1; /* 4kb granularity */
1502 cs->limit = 0xffffffff; /* 4GB limit */
1503 cs->type = 0x0b; /* Read, Execute, Accessed */
1505 cs->dpl = 0; /* will be adjusted later */
1510 ss->base = 0; /* flat segment */
1511 ss->limit = 0xffffffff; /* 4GB limit */
1512 ss->g = 1; /* 4kb granularity */
1514 ss->type = 0x03; /* Read/Write, Accessed */
1515 ss->db = 1; /* 32bit stack segment */
1521 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1523 struct decode_cache *c = &ctxt->decode;
1524 struct kvm_segment cs, ss;
1527 /* syscall is not available in real mode */
1528 if (c->lock_prefix || ctxt->mode == X86EMUL_MODE_REAL
1529 || ctxt->mode == X86EMUL_MODE_VM86)
1532 setup_syscalls_segments(ctxt, &cs, &ss);
1534 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1536 cs.selector = (u16)(msr_data & 0xfffc);
1537 ss.selector = (u16)(msr_data + 8);
1539 if (is_long_mode(ctxt->vcpu)) {
1543 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1544 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1546 c->regs[VCPU_REGS_RCX] = c->eip;
1547 if (is_long_mode(ctxt->vcpu)) {
1548 #ifdef CONFIG_X86_64
1549 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1551 kvm_x86_ops->get_msr(ctxt->vcpu,
1552 ctxt->mode == X86EMUL_MODE_PROT64 ?
1553 MSR_LSTAR : MSR_CSTAR, &msr_data);
1556 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1557 ctxt->eflags &= ~(msr_data | EFLG_RF);
1561 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1562 c->eip = (u32)msr_data;
1564 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1571 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1573 struct decode_cache *c = &ctxt->decode;
1574 struct kvm_segment cs, ss;
1577 /* inject #UD if LOCK prefix is used */
1581 /* inject #GP if in real mode */
1582 if (ctxt->mode == X86EMUL_MODE_REAL) {
1583 kvm_inject_gp(ctxt->vcpu, 0);
1587 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1588 * Therefore, we inject an #UD.
1590 if (ctxt->mode == X86EMUL_MODE_PROT64)
1593 setup_syscalls_segments(ctxt, &cs, &ss);
1595 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1596 switch (ctxt->mode) {
1597 case X86EMUL_MODE_PROT32:
1598 if ((msr_data & 0xfffc) == 0x0) {
1599 kvm_inject_gp(ctxt->vcpu, 0);
1603 case X86EMUL_MODE_PROT64:
1604 if (msr_data == 0x0) {
1605 kvm_inject_gp(ctxt->vcpu, 0);
1611 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1612 cs.selector = (u16)msr_data;
1613 cs.selector &= ~SELECTOR_RPL_MASK;
1614 ss.selector = cs.selector + 8;
1615 ss.selector &= ~SELECTOR_RPL_MASK;
1616 if (ctxt->mode == X86EMUL_MODE_PROT64
1617 || is_long_mode(ctxt->vcpu)) {
1622 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1623 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1625 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1628 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1629 c->regs[VCPU_REGS_RSP] = msr_data;
1635 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1637 struct decode_cache *c = &ctxt->decode;
1638 struct kvm_segment cs, ss;
1642 /* inject #UD if LOCK prefix is used */
1646 /* inject #GP if in real mode or Virtual 8086 mode */
1647 if (ctxt->mode == X86EMUL_MODE_REAL ||
1648 ctxt->mode == X86EMUL_MODE_VM86) {
1649 kvm_inject_gp(ctxt->vcpu, 0);
1653 /* sysexit must be called from CPL 0 */
1654 if (kvm_x86_ops->get_cpl(ctxt->vcpu) != 0) {
1655 kvm_inject_gp(ctxt->vcpu, 0);
1659 setup_syscalls_segments(ctxt, &cs, &ss);
1661 if ((c->rex_prefix & 0x8) != 0x0)
1662 usermode = X86EMUL_MODE_PROT64;
1664 usermode = X86EMUL_MODE_PROT32;
1668 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1670 case X86EMUL_MODE_PROT32:
1671 cs.selector = (u16)(msr_data + 16);
1672 if ((msr_data & 0xfffc) == 0x0) {
1673 kvm_inject_gp(ctxt->vcpu, 0);
1676 ss.selector = (u16)(msr_data + 24);
1678 case X86EMUL_MODE_PROT64:
1679 cs.selector = (u16)(msr_data + 32);
1680 if (msr_data == 0x0) {
1681 kvm_inject_gp(ctxt->vcpu, 0);
1684 ss.selector = cs.selector + 8;
1689 cs.selector |= SELECTOR_RPL_MASK;
1690 ss.selector |= SELECTOR_RPL_MASK;
1692 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1693 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1695 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1696 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1702 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1704 unsigned long memop = 0;
1706 unsigned long saved_eip = 0;
1707 struct decode_cache *c = &ctxt->decode;
1712 ctxt->interruptibility = 0;
1714 /* Shadow copy of register state. Committed on successful emulation.
1715 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1719 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1722 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1723 memop = c->modrm_ea;
1725 if (c->rep_prefix && (c->d & String)) {
1726 /* All REP prefixes have the same first termination condition */
1727 if (c->regs[VCPU_REGS_RCX] == 0) {
1728 kvm_rip_write(ctxt->vcpu, c->eip);
1731 /* The second termination condition only applies for REPE
1732 * and REPNE. Test if the repeat string operation prefix is
1733 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1734 * corresponding termination condition according to:
1735 * - if REPE/REPZ and ZF = 0 then done
1736 * - if REPNE/REPNZ and ZF = 1 then done
1738 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1739 (c->b == 0xae) || (c->b == 0xaf)) {
1740 if ((c->rep_prefix == REPE_PREFIX) &&
1741 ((ctxt->eflags & EFLG_ZF) == 0)) {
1742 kvm_rip_write(ctxt->vcpu, c->eip);
1745 if ((c->rep_prefix == REPNE_PREFIX) &&
1746 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1747 kvm_rip_write(ctxt->vcpu, c->eip);
1751 c->regs[VCPU_REGS_RCX]--;
1752 c->eip = kvm_rip_read(ctxt->vcpu);
1755 if (c->src.type == OP_MEM) {
1756 c->src.ptr = (unsigned long *)memop;
1758 rc = ops->read_emulated((unsigned long)c->src.ptr,
1762 if (rc != X86EMUL_CONTINUE)
1764 c->src.orig_val = c->src.val;
1767 if ((c->d & DstMask) == ImplicitOps)
1771 if (c->dst.type == OP_MEM) {
1772 c->dst.ptr = (unsigned long *)memop;
1773 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1776 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1778 c->dst.ptr = (void *)c->dst.ptr +
1779 (c->src.val & mask) / 8;
1781 if (!(c->d & Mov)) {
1782 /* optimisation - avoid slow emulated read */
1783 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1787 if (rc != X86EMUL_CONTINUE)
1791 c->dst.orig_val = c->dst.val;
1801 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1803 case 0x06: /* push es */
1804 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1806 case 0x07: /* pop es */
1807 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1813 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1815 case 0x0e: /* push cs */
1816 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1820 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1822 case 0x16: /* push ss */
1823 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1825 case 0x17: /* pop ss */
1826 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1832 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1834 case 0x1e: /* push ds */
1835 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1837 case 0x1f: /* pop ds */
1838 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1844 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1848 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1852 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1856 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1858 case 0x40 ... 0x47: /* inc r16/r32 */
1859 emulate_1op("inc", c->dst, ctxt->eflags);
1861 case 0x48 ... 0x4f: /* dec r16/r32 */
1862 emulate_1op("dec", c->dst, ctxt->eflags);
1864 case 0x50 ... 0x57: /* push reg */
1867 case 0x58 ... 0x5f: /* pop reg */
1869 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1873 case 0x60: /* pusha */
1874 emulate_pusha(ctxt);
1876 case 0x61: /* popa */
1877 rc = emulate_popa(ctxt, ops);
1881 case 0x63: /* movsxd */
1882 if (ctxt->mode != X86EMUL_MODE_PROT64)
1883 goto cannot_emulate;
1884 c->dst.val = (s32) c->src.val;
1886 case 0x68: /* push imm */
1887 case 0x6a: /* push imm8 */
1890 case 0x6c: /* insb */
1891 case 0x6d: /* insw/insd */
1892 if (kvm_emulate_pio_string(ctxt->vcpu,
1894 (c->d & ByteOp) ? 1 : c->op_bytes,
1896 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1897 (ctxt->eflags & EFLG_DF),
1898 register_address(c, es_base(ctxt),
1899 c->regs[VCPU_REGS_RDI]),
1901 c->regs[VCPU_REGS_RDX]) == 0) {
1906 case 0x6e: /* outsb */
1907 case 0x6f: /* outsw/outsd */
1908 if (kvm_emulate_pio_string(ctxt->vcpu,
1910 (c->d & ByteOp) ? 1 : c->op_bytes,
1912 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1913 (ctxt->eflags & EFLG_DF),
1915 seg_override_base(ctxt, c),
1916 c->regs[VCPU_REGS_RSI]),
1918 c->regs[VCPU_REGS_RDX]) == 0) {
1923 case 0x70 ... 0x7f: /* jcc (short) */
1924 if (test_cc(c->b, ctxt->eflags))
1925 jmp_rel(c, c->src.val);
1927 case 0x80 ... 0x83: /* Grp1 */
1928 switch (c->modrm_reg) {
1948 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1950 case 0x86 ... 0x87: /* xchg */
1952 /* Write back the register source. */
1953 switch (c->dst.bytes) {
1955 *(u8 *) c->src.ptr = (u8) c->dst.val;
1958 *(u16 *) c->src.ptr = (u16) c->dst.val;
1961 *c->src.ptr = (u32) c->dst.val;
1962 break; /* 64b reg: zero-extend */
1964 *c->src.ptr = c->dst.val;
1968 * Write back the memory destination with implicit LOCK
1971 c->dst.val = c->src.val;
1974 case 0x88 ... 0x8b: /* mov */
1976 case 0x8c: { /* mov r/m, sreg */
1977 struct kvm_segment segreg;
1979 if (c->modrm_reg <= 5)
1980 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
1982 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
1984 goto cannot_emulate;
1986 c->dst.val = segreg.selector;
1989 case 0x8d: /* lea r16/r32, m */
1990 c->dst.val = c->modrm_ea;
1992 case 0x8e: { /* mov seg, r/m16 */
1998 if (c->modrm_reg == VCPU_SREG_SS)
1999 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
2001 if (c->modrm_reg <= 5) {
2002 type_bits = (c->modrm_reg == 1) ? 9 : 1;
2003 err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
2004 type_bits, c->modrm_reg);
2006 printk(KERN_INFO "Invalid segreg in modrm byte 0x%02x\n",
2008 goto cannot_emulate;
2012 goto cannot_emulate;
2014 c->dst.type = OP_NONE; /* Disable writeback. */
2017 case 0x8f: /* pop (sole member of Grp1a) */
2018 rc = emulate_grp1a(ctxt, ops);
2022 case 0x90: /* nop / xchg r8,rax */
2023 if (!(c->rex_prefix & 1)) { /* nop */
2024 c->dst.type = OP_NONE;
2027 case 0x91 ... 0x97: /* xchg reg,rax */
2028 c->src.type = c->dst.type = OP_REG;
2029 c->src.bytes = c->dst.bytes = c->op_bytes;
2030 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2031 c->src.val = *(c->src.ptr);
2033 case 0x9c: /* pushf */
2034 c->src.val = (unsigned long) ctxt->eflags;
2037 case 0x9d: /* popf */
2038 c->dst.type = OP_REG;
2039 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2040 c->dst.bytes = c->op_bytes;
2041 goto pop_instruction;
2042 case 0xa0 ... 0xa1: /* mov */
2043 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2044 c->dst.val = c->src.val;
2046 case 0xa2 ... 0xa3: /* mov */
2047 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2049 case 0xa4 ... 0xa5: /* movs */
2050 c->dst.type = OP_MEM;
2051 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2052 c->dst.ptr = (unsigned long *)register_address(c,
2054 c->regs[VCPU_REGS_RDI]);
2055 rc = ops->read_emulated(register_address(c,
2056 seg_override_base(ctxt, c),
2057 c->regs[VCPU_REGS_RSI]),
2059 c->dst.bytes, ctxt->vcpu);
2060 if (rc != X86EMUL_CONTINUE)
2062 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2063 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2065 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2066 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2069 case 0xa6 ... 0xa7: /* cmps */
2070 c->src.type = OP_NONE; /* Disable writeback. */
2071 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2072 c->src.ptr = (unsigned long *)register_address(c,
2073 seg_override_base(ctxt, c),
2074 c->regs[VCPU_REGS_RSI]);
2075 rc = ops->read_emulated((unsigned long)c->src.ptr,
2079 if (rc != X86EMUL_CONTINUE)
2082 c->dst.type = OP_NONE; /* Disable writeback. */
2083 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2084 c->dst.ptr = (unsigned long *)register_address(c,
2086 c->regs[VCPU_REGS_RDI]);
2087 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2091 if (rc != X86EMUL_CONTINUE)
2094 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2096 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2098 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2099 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2101 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2102 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2106 case 0xaa ... 0xab: /* stos */
2107 c->dst.type = OP_MEM;
2108 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2109 c->dst.ptr = (unsigned long *)register_address(c,
2111 c->regs[VCPU_REGS_RDI]);
2112 c->dst.val = c->regs[VCPU_REGS_RAX];
2113 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2114 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2117 case 0xac ... 0xad: /* lods */
2118 c->dst.type = OP_REG;
2119 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2120 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2121 rc = ops->read_emulated(register_address(c,
2122 seg_override_base(ctxt, c),
2123 c->regs[VCPU_REGS_RSI]),
2127 if (rc != X86EMUL_CONTINUE)
2129 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2130 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2133 case 0xae ... 0xaf: /* scas */
2134 DPRINTF("Urk! I don't handle SCAS.\n");
2135 goto cannot_emulate;
2136 case 0xb0 ... 0xbf: /* mov r, imm */
2141 case 0xc3: /* ret */
2142 c->dst.type = OP_REG;
2143 c->dst.ptr = &c->eip;
2144 c->dst.bytes = c->op_bytes;
2145 goto pop_instruction;
2146 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2148 c->dst.val = c->src.val;
2150 case 0xcb: /* ret far */
2151 rc = emulate_ret_far(ctxt, ops);
2155 case 0xd0 ... 0xd1: /* Grp2 */
2159 case 0xd2 ... 0xd3: /* Grp2 */
2160 c->src.val = c->regs[VCPU_REGS_RCX];
2163 case 0xe4: /* inb */
2168 case 0xe6: /* outb */
2169 case 0xe7: /* out */
2173 case 0xe8: /* call (near) */ {
2174 long int rel = c->src.val;
2175 c->src.val = (unsigned long) c->eip;
2180 case 0xe9: /* jmp rel */
2182 case 0xea: /* jmp far */
2183 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val, 9,
2184 VCPU_SREG_CS) < 0) {
2185 DPRINTF("jmp far: Failed to load CS descriptor\n");
2186 goto cannot_emulate;
2189 c->eip = c->src.val;
2192 jmp: /* jmp rel short */
2193 jmp_rel(c, c->src.val);
2194 c->dst.type = OP_NONE; /* Disable writeback. */
2196 case 0xec: /* in al,dx */
2197 case 0xed: /* in (e/r)ax,dx */
2198 port = c->regs[VCPU_REGS_RDX];
2201 case 0xee: /* out al,dx */
2202 case 0xef: /* out (e/r)ax,dx */
2203 port = c->regs[VCPU_REGS_RDX];
2205 do_io: if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2206 (c->d & ByteOp) ? 1 : c->op_bytes,
2209 goto cannot_emulate;
2212 case 0xf4: /* hlt */
2213 ctxt->vcpu->arch.halt_request = 1;
2215 case 0xf5: /* cmc */
2216 /* complement carry flag from eflags reg */
2217 ctxt->eflags ^= EFLG_CF;
2218 c->dst.type = OP_NONE; /* Disable writeback. */
2220 case 0xf6 ... 0xf7: /* Grp3 */
2221 rc = emulate_grp3(ctxt, ops);
2225 case 0xf8: /* clc */
2226 ctxt->eflags &= ~EFLG_CF;
2227 c->dst.type = OP_NONE; /* Disable writeback. */
2229 case 0xfa: /* cli */
2230 ctxt->eflags &= ~X86_EFLAGS_IF;
2231 c->dst.type = OP_NONE; /* Disable writeback. */
2233 case 0xfb: /* sti */
2234 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2235 ctxt->eflags |= X86_EFLAGS_IF;
2236 c->dst.type = OP_NONE; /* Disable writeback. */
2238 case 0xfc: /* cld */
2239 ctxt->eflags &= ~EFLG_DF;
2240 c->dst.type = OP_NONE; /* Disable writeback. */
2242 case 0xfd: /* std */
2243 ctxt->eflags |= EFLG_DF;
2244 c->dst.type = OP_NONE; /* Disable writeback. */
2246 case 0xfe ... 0xff: /* Grp4/Grp5 */
2247 rc = emulate_grp45(ctxt, ops);
2254 rc = writeback(ctxt, ops);
2258 /* Commit shadow register state. */
2259 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2260 kvm_rip_write(ctxt->vcpu, c->eip);
2263 if (rc == X86EMUL_UNHANDLEABLE) {
2271 case 0x01: /* lgdt, lidt, lmsw */
2272 switch (c->modrm_reg) {
2274 unsigned long address;
2276 case 0: /* vmcall */
2277 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2278 goto cannot_emulate;
2280 rc = kvm_fix_hypercall(ctxt->vcpu);
2284 /* Let the processor re-execute the fixed hypercall */
2285 c->eip = kvm_rip_read(ctxt->vcpu);
2286 /* Disable writeback. */
2287 c->dst.type = OP_NONE;
2290 rc = read_descriptor(ctxt, ops, c->src.ptr,
2291 &size, &address, c->op_bytes);
2294 realmode_lgdt(ctxt->vcpu, size, address);
2295 /* Disable writeback. */
2296 c->dst.type = OP_NONE;
2298 case 3: /* lidt/vmmcall */
2299 if (c->modrm_mod == 3) {
2300 switch (c->modrm_rm) {
2302 rc = kvm_fix_hypercall(ctxt->vcpu);
2307 goto cannot_emulate;
2310 rc = read_descriptor(ctxt, ops, c->src.ptr,
2315 realmode_lidt(ctxt->vcpu, size, address);
2317 /* Disable writeback. */
2318 c->dst.type = OP_NONE;
2322 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2325 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2327 c->dst.type = OP_NONE;
2330 emulate_invlpg(ctxt->vcpu, memop);
2331 /* Disable writeback. */
2332 c->dst.type = OP_NONE;
2335 goto cannot_emulate;
2338 case 0x05: /* syscall */
2339 if (emulate_syscall(ctxt) == -1)
2340 goto cannot_emulate;
2345 emulate_clts(ctxt->vcpu);
2346 c->dst.type = OP_NONE;
2348 case 0x08: /* invd */
2349 case 0x09: /* wbinvd */
2350 case 0x0d: /* GrpP (prefetch) */
2351 case 0x18: /* Grp16 (prefetch/nop) */
2352 c->dst.type = OP_NONE;
2354 case 0x20: /* mov cr, reg */
2355 if (c->modrm_mod != 3)
2356 goto cannot_emulate;
2357 c->regs[c->modrm_rm] =
2358 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2359 c->dst.type = OP_NONE; /* no writeback */
2361 case 0x21: /* mov from dr to reg */
2362 if (c->modrm_mod != 3)
2363 goto cannot_emulate;
2364 rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
2366 goto cannot_emulate;
2367 c->dst.type = OP_NONE; /* no writeback */
2369 case 0x22: /* mov reg, cr */
2370 if (c->modrm_mod != 3)
2371 goto cannot_emulate;
2372 realmode_set_cr(ctxt->vcpu,
2373 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2374 c->dst.type = OP_NONE;
2376 case 0x23: /* mov from reg to dr */
2377 if (c->modrm_mod != 3)
2378 goto cannot_emulate;
2379 rc = emulator_set_dr(ctxt, c->modrm_reg,
2380 c->regs[c->modrm_rm]);
2382 goto cannot_emulate;
2383 c->dst.type = OP_NONE; /* no writeback */
2387 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2388 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2389 rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
2391 kvm_inject_gp(ctxt->vcpu, 0);
2392 c->eip = kvm_rip_read(ctxt->vcpu);
2394 rc = X86EMUL_CONTINUE;
2395 c->dst.type = OP_NONE;
2399 rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
2401 kvm_inject_gp(ctxt->vcpu, 0);
2402 c->eip = kvm_rip_read(ctxt->vcpu);
2404 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2405 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2407 rc = X86EMUL_CONTINUE;
2408 c->dst.type = OP_NONE;
2410 case 0x34: /* sysenter */
2411 if (emulate_sysenter(ctxt) == -1)
2412 goto cannot_emulate;
2416 case 0x35: /* sysexit */
2417 if (emulate_sysexit(ctxt) == -1)
2418 goto cannot_emulate;
2422 case 0x40 ... 0x4f: /* cmov */
2423 c->dst.val = c->dst.orig_val = c->src.val;
2424 if (!test_cc(c->b, ctxt->eflags))
2425 c->dst.type = OP_NONE; /* no writeback */
2427 case 0x80 ... 0x8f: /* jnz rel, etc*/
2428 if (test_cc(c->b, ctxt->eflags))
2429 jmp_rel(c, c->src.val);
2430 c->dst.type = OP_NONE;
2432 case 0xa0: /* push fs */
2433 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2435 case 0xa1: /* pop fs */
2436 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2442 c->dst.type = OP_NONE;
2443 /* only subword offset */
2444 c->src.val &= (c->dst.bytes << 3) - 1;
2445 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2447 case 0xa4: /* shld imm8, r, r/m */
2448 case 0xa5: /* shld cl, r, r/m */
2449 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2451 case 0xa8: /* push gs */
2452 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2454 case 0xa9: /* pop gs */
2455 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2461 /* only subword offset */
2462 c->src.val &= (c->dst.bytes << 3) - 1;
2463 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2465 case 0xac: /* shrd imm8, r, r/m */
2466 case 0xad: /* shrd cl, r, r/m */
2467 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2469 case 0xae: /* clflush */
2471 case 0xb0 ... 0xb1: /* cmpxchg */
2473 * Save real source value, then compare EAX against
2476 c->src.orig_val = c->src.val;
2477 c->src.val = c->regs[VCPU_REGS_RAX];
2478 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2479 if (ctxt->eflags & EFLG_ZF) {
2480 /* Success: write back to memory. */
2481 c->dst.val = c->src.orig_val;
2483 /* Failure: write the value we saw to EAX. */
2484 c->dst.type = OP_REG;
2485 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2490 /* only subword offset */
2491 c->src.val &= (c->dst.bytes << 3) - 1;
2492 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2494 case 0xb6 ... 0xb7: /* movzx */
2495 c->dst.bytes = c->op_bytes;
2496 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2499 case 0xba: /* Grp8 */
2500 switch (c->modrm_reg & 3) {
2513 /* only subword offset */
2514 c->src.val &= (c->dst.bytes << 3) - 1;
2515 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2517 case 0xbe ... 0xbf: /* movsx */
2518 c->dst.bytes = c->op_bytes;
2519 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2522 case 0xc3: /* movnti */
2523 c->dst.bytes = c->op_bytes;
2524 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2527 case 0xc7: /* Grp9 (cmpxchg8b) */
2528 rc = emulate_grp9(ctxt, ops, memop);
2531 c->dst.type = OP_NONE;
2537 DPRINTF("Cannot emulate %02x\n", c->b);