KVM: x86 emulator: fix 0f 01 /5 emulation
[safe/jmp/linux-2.6] / arch / x86 / kvm / emulate.c
index c280c23..7c7debb 100644 (file)
@@ -85,6 +85,9 @@
 #define Src2ImmByte (2<<29)
 #define Src2One     (3<<29)
 #define Src2Imm16   (4<<29)
+#define Src2Mem16   (5<<29) /* Used for Ep encoding. First argument has to be
+                              in memory and second argument is located
+                              immediately after the first one in memory. */
 #define Src2Mask    (7<<29)
 
 enum {
@@ -343,7 +346,8 @@ static u32 group_table[] = {
        [Group5*8] =
        DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
        SrcMem | ModRM | Stack, 0,
-       SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
+       SrcMem | ModRM | Stack, SrcMem | ModRM | Src2Mem16 | ImplicitOps,
+       SrcMem | ModRM | Stack, 0,
        [Group7*8] =
        0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
        SrcNone | ModRM | DstMem | Mov, 0,
@@ -358,9 +362,9 @@ static u32 group_table[] = {
 
 static u32 group2_table[] = {
        [Group7*8] =
-       SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM,
+       SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM | Priv,
        SrcNone | ModRM | DstMem | Mov, 0,
-       SrcMem16 | ModRM | Mov, 0,
+       SrcMem16 | ModRM | Mov | Priv, 0,
        [Group9*8] =
        0, 0, 0, 0, 0, 0, 0, 0,
 };
@@ -562,7 +566,7 @@ static u32 group2_table[] = {
 #define insn_fetch(_type, _size, _eip)                                  \
 ({     unsigned long _x;                                               \
        rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size));            \
-       if (rc != 0)                                                    \
+       if (rc != X86EMUL_CONTINUE)                                     \
                goto done;                                              \
        (_eip) += (_size);                                              \
        (_type)_x;                                                      \
@@ -647,31 +651,31 @@ static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
        if (linear < fc->start || linear >= fc->end) {
                size = min(15UL, PAGE_SIZE - offset_in_page(linear));
                rc = ops->fetch(linear, fc->data, size, ctxt->vcpu, NULL);
-               if (rc)
+               if (rc != X86EMUL_CONTINUE)
                        return rc;
                fc->start = linear;
                fc->end = linear + size;
        }
        *dest = fc->data[linear - fc->start];
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
                         struct x86_emulate_ops *ops,
                         unsigned long eip, void *dest, unsigned size)
 {
-       int rc = 0;
+       int rc;
 
        /* x86 instructions are limited to 15 bytes. */
-       if (eip + size - ctxt->decode.eip_orig > 15)
+       if (eip + size - ctxt->eip > 15)
                return X86EMUL_UNHANDLEABLE;
        eip += ctxt->cs_base;
        while (size--) {
                rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
-               if (rc)
+               if (rc != X86EMUL_CONTINUE)
                        return rc;
        }
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
 /*
@@ -702,7 +706,7 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt,
        *address = 0;
        rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
                           ctxt->vcpu, NULL);
-       if (rc)
+       if (rc != X86EMUL_CONTINUE)
                return rc;
        rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
                           ctxt->vcpu, NULL);
@@ -782,7 +786,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
        struct decode_cache *c = &ctxt->decode;
        u8 sib;
        int index_reg = 0, base_reg = 0, scale;
-       int rc = 0;
+       int rc = X86EMUL_CONTINUE;
 
        if (c->rex_prefix) {
                c->modrm_reg = (c->rex_prefix & 4) << 1;        /* REX.R */
@@ -895,7 +899,7 @@ static int decode_abs(struct x86_emulate_ctxt *ctxt,
                      struct x86_emulate_ops *ops)
 {
        struct decode_cache *c = &ctxt->decode;
-       int rc = 0;
+       int rc = X86EMUL_CONTINUE;
 
        switch (c->ad_bytes) {
        case 2:
@@ -916,14 +920,14 @@ int
 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 {
        struct decode_cache *c = &ctxt->decode;
-       int rc = 0;
+       int rc = X86EMUL_CONTINUE;
        int mode = ctxt->mode;
        int def_op_bytes, def_ad_bytes, group;
 
        /* Shadow copy of register state. Committed on successful emulation. */
 
        memset(c, 0, sizeof(struct decode_cache));
-       c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
+       c->eip = ctxt->eip;
        ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
        memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
 
@@ -1015,11 +1019,6 @@ done_prefixes:
                }
        }
 
-       if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
-               kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");
-               return -1;
-       }
-
        if (c->d & Group) {
                group = c->d & GroupMask;
                c->modrm = insn_fetch(u8, 1, c->eip);
@@ -1046,7 +1045,7 @@ done_prefixes:
                rc = decode_modrm(ctxt, ops);
        else if (c->d & MemAbs)
                rc = decode_abs(ctxt, ops);
-       if (rc)
+       if (rc != X86EMUL_CONTINUE)
                goto done;
 
        if (!c->has_seg_override)
@@ -1168,6 +1167,10 @@ done_prefixes:
                c->src2.bytes = 1;
                c->src2.val = 1;
                break;
+       case Src2Mem16:
+               c->src2.bytes = 2;
+               c->src2.type = OP_MEM;
+               break;
        }
 
        /* Decode and fetch the destination operand: register or memory. */
@@ -1191,9 +1194,9 @@ done_prefixes:
                break;
        case DstAcc:
                c->dst.type = OP_REG;
-               c->dst.bytes = c->op_bytes;
+               c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
                c->dst.ptr = &c->regs[VCPU_REGS_RAX];
-               switch (c->op_bytes) {
+               switch (c->dst.bytes) {
                        case 1:
                                c->dst.val = *(u8 *)c->dst.ptr;
                                break;
@@ -1203,6 +1206,9 @@ done_prefixes:
                        case 4:
                                c->dst.val = *(u32 *)c->dst.ptr;
                                break;
+                       case 8:
+                               c->dst.val = *(u64 *)c->dst.ptr;
+                               break;
                }
                c->dst.orig_val = c->dst.val;
                break;
@@ -1251,7 +1257,7 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
        int rc;
        unsigned long val, change_mask;
        int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
-       int cpl = kvm_x86_ops->get_cpl(ctxt->vcpu);
+       int cpl = ops->cpl(ctxt->vcpu);
 
        rc = emulate_pop(ctxt, ops, &val, len);
        if (rc != X86EMUL_CONTINUE)
@@ -1306,10 +1312,10 @@ static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
        int rc;
 
        rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
-       if (rc != 0)
+       if (rc != X86EMUL_CONTINUE)
                return rc;
 
-       rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, 1, seg);
+       rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, seg);
        return rc;
 }
 
@@ -1332,7 +1338,7 @@ static int emulate_popa(struct x86_emulate_ctxt *ctxt,
                        struct x86_emulate_ops *ops)
 {
        struct decode_cache *c = &ctxt->decode;
-       int rc = 0;
+       int rc = X86EMUL_CONTINUE;
        int reg = VCPU_REGS_RDI;
 
        while (reg >= VCPU_REGS_RAX) {
@@ -1343,7 +1349,7 @@ static int emulate_popa(struct x86_emulate_ctxt *ctxt,
                }
 
                rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        break;
                --reg;
        }
@@ -1354,12 +1360,8 @@ static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
                                struct x86_emulate_ops *ops)
 {
        struct decode_cache *c = &ctxt->decode;
-       int rc;
 
-       rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
-       if (rc != 0)
-               return rc;
-       return 0;
+       return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
 }
 
 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
@@ -1395,7 +1397,7 @@ static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
                               struct x86_emulate_ops *ops)
 {
        struct decode_cache *c = &ctxt->decode;
-       int rc = 0;
+       int rc = X86EMUL_CONTINUE;
 
        switch (c->modrm_reg) {
        case 0 ... 1:   /* test */
@@ -1442,7 +1444,7 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
                emulate_push(ctxt);
                break;
        }
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
@@ -1473,7 +1475,7 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
                        return rc;
                ctxt->eflags |= EFLG_ZF;
        }
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
@@ -1484,14 +1486,14 @@ static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
        unsigned long cs;
 
        rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
-       if (rc)
+       if (rc != X86EMUL_CONTINUE)
                return rc;
        if (c->op_bytes == 4)
                c->eip = (u32)c->eip;
        rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
-       if (rc)
+       if (rc != X86EMUL_CONTINUE)
                return rc;
-       rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, 1, VCPU_SREG_CS);
+       rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, VCPU_SREG_CS);
        return rc;
 }
 
@@ -1544,7 +1546,7 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt,
        default:
                break;
        }
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
@@ -1599,7 +1601,7 @@ emulate_syscall(struct x86_emulate_ctxt *ctxt)
 
        /* syscall is not available in real mode */
        if (ctxt->mode == X86EMUL_MODE_REAL || ctxt->mode == X86EMUL_MODE_VM86)
-               return -1;
+               return X86EMUL_UNHANDLEABLE;
 
        setup_syscalls_segments(ctxt, &cs, &ss);
 
@@ -1636,7 +1638,7 @@ emulate_syscall(struct x86_emulate_ctxt *ctxt)
                ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
        }
 
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
 static int
@@ -1649,14 +1651,14 @@ emulate_sysenter(struct x86_emulate_ctxt *ctxt)
        /* inject #GP if in real mode */
        if (ctxt->mode == X86EMUL_MODE_REAL) {
                kvm_inject_gp(ctxt->vcpu, 0);
-               return -1;
+               return X86EMUL_UNHANDLEABLE;
        }
 
        /* XXX sysenter/sysexit have not been tested in 64bit mode.
        * Therefore, we inject an #UD.
        */
        if (ctxt->mode == X86EMUL_MODE_PROT64)
-               return -1;
+               return X86EMUL_UNHANDLEABLE;
 
        setup_syscalls_segments(ctxt, &cs, &ss);
 
@@ -1665,13 +1667,13 @@ emulate_sysenter(struct x86_emulate_ctxt *ctxt)
        case X86EMUL_MODE_PROT32:
                if ((msr_data & 0xfffc) == 0x0) {
                        kvm_inject_gp(ctxt->vcpu, 0);
-                       return -1;
+                       return X86EMUL_PROPAGATE_FAULT;
                }
                break;
        case X86EMUL_MODE_PROT64:
                if (msr_data == 0x0) {
                        kvm_inject_gp(ctxt->vcpu, 0);
-                       return -1;
+                       return X86EMUL_PROPAGATE_FAULT;
                }
                break;
        }
@@ -1696,7 +1698,7 @@ emulate_sysenter(struct x86_emulate_ctxt *ctxt)
        kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
        c->regs[VCPU_REGS_RSP] = msr_data;
 
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
 static int
@@ -1711,7 +1713,7 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt)
        if (ctxt->mode == X86EMUL_MODE_REAL ||
            ctxt->mode == X86EMUL_MODE_VM86) {
                kvm_inject_gp(ctxt->vcpu, 0);
-               return -1;
+               return X86EMUL_UNHANDLEABLE;
        }
 
        setup_syscalls_segments(ctxt, &cs, &ss);
@@ -1729,7 +1731,7 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt)
                cs.selector = (u16)(msr_data + 16);
                if ((msr_data & 0xfffc) == 0x0) {
                        kvm_inject_gp(ctxt->vcpu, 0);
-                       return -1;
+                       return X86EMUL_PROPAGATE_FAULT;
                }
                ss.selector = (u16)(msr_data + 24);
                break;
@@ -1737,7 +1739,7 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt)
                cs.selector = (u16)(msr_data + 32);
                if (msr_data == 0x0) {
                        kvm_inject_gp(ctxt->vcpu, 0);
-                       return -1;
+                       return X86EMUL_PROPAGATE_FAULT;
                }
                ss.selector = cs.selector + 8;
                cs.db = 0;
@@ -1753,10 +1755,11 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt)
        c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
        c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
 
-       return 0;
+       return X86EMUL_CONTINUE;
 }
 
-static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
+static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
+                             struct x86_emulate_ops *ops)
 {
        int iopl;
        if (ctxt->mode == X86EMUL_MODE_REAL)
@@ -1764,7 +1767,7 @@ static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
        if (ctxt->mode == X86EMUL_MODE_VM86)
                return true;
        iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
-       return kvm_x86_ops->get_cpl(ctxt->vcpu) > iopl;
+       return ops->cpl(ctxt->vcpu) > iopl;
 }
 
 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
@@ -1801,7 +1804,7 @@ static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
                                 struct x86_emulate_ops *ops,
                                 u16 port, u16 len)
 {
-       if (emulator_bad_iopl(ctxt))
+       if (emulator_bad_iopl(ctxt, ops))
                if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
                        return false;
        return true;
@@ -1816,7 +1819,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
        struct decode_cache *c = &ctxt->decode;
        unsigned int port;
        int io_dir_in;
-       int rc = 0;
+       int rc = X86EMUL_CONTINUE;
 
        ctxt->interruptibility = 0;
 
@@ -1828,6 +1831,11 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
        memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
        saved_eip = c->eip;
 
+       if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
+               kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
+               goto done;
+       }
+
        /* LOCK prefix is allowed only with some instructions */
        if (c->lock_prefix && !(c->d & Lock)) {
                kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
@@ -1835,7 +1843,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
        }
 
        /* Privileged instruction can be executed only in CPL=0 */
-       if ((c->d & Priv) && kvm_x86_ops->get_cpl(ctxt->vcpu)) {
+       if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
                kvm_inject_gp(ctxt->vcpu, 0);
                goto done;
        }
@@ -1845,7 +1853,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 
        if (c->rep_prefix && (c->d & String)) {
                /* All REP prefixes have the same first termination condition */
-               if (c->regs[VCPU_REGS_RCX] == 0) {
+               if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
                        kvm_rip_write(ctxt->vcpu, c->eip);
                        goto done;
                }
@@ -1869,8 +1877,8 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
                                goto done;
                        }
                }
-               c->regs[VCPU_REGS_RCX]--;
-               c->eip = kvm_rip_read(ctxt->vcpu);
+               register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
+               c->eip = ctxt->eip;
        }
 
        if (c->src.type == OP_MEM) {
@@ -1885,6 +1893,17 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
                c->src.orig_val = c->src.val;
        }
 
+       if (c->src2.type == OP_MEM) {
+               c->src2.ptr = (unsigned long *)(memop + c->src.bytes);
+               c->src2.val = 0;
+               rc = ops->read_emulated((unsigned long)c->src2.ptr,
+                                       &c->src2.val,
+                                       c->src2.bytes,
+                                       ctxt->vcpu);
+               if (rc != X86EMUL_CONTINUE)
+                       goto done;
+       }
+
        if ((c->d & DstMask) == ImplicitOps)
                goto special_insn;
 
@@ -1926,7 +1945,7 @@ special_insn:
                break;
        case 0x07:              /* pop es */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0x08 ... 0x0d:
@@ -1945,7 +1964,7 @@ special_insn:
                break;
        case 0x17:              /* pop ss */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0x18 ... 0x1d:
@@ -1957,7 +1976,7 @@ special_insn:
                break;
        case 0x1f:              /* pop ds */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0x20 ... 0x25:
@@ -1988,7 +2007,7 @@ special_insn:
        case 0x58 ... 0x5f: /* pop reg */
        pop_instruction:
                rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0x60:      /* pusha */
@@ -1996,7 +2015,7 @@ special_insn:
                break;
        case 0x61:      /* popa */
                rc = emulate_popa(ctxt, ops);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0x63:              /* movsxd */
@@ -2107,12 +2126,11 @@ special_insn:
        case 0x8c: { /* mov r/m, sreg */
                struct kvm_segment segreg;
 
-               if (c->modrm_reg <= 5)
+               if (c->modrm_reg <= VCPU_SREG_GS)
                        kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
                else {
-                       printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
-                              c->modrm);
-                       goto cannot_emulate;
+                       kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
+                       goto done;
                }
                c->dst.val = segreg.selector;
                break;
@@ -2122,32 +2140,26 @@ special_insn:
                break;
        case 0x8e: { /* mov seg, r/m16 */
                uint16_t sel;
-               int type_bits;
-               int err;
 
                sel = c->src.val;
-               if (c->modrm_reg == VCPU_SREG_SS)
-                       toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
 
-               if (c->modrm_reg <= 5) {
-                       type_bits = (c->modrm_reg == 1) ? 9 : 1;
-                       err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
-                                                         type_bits, c->modrm_reg);
-               } else {
-                       printk(KERN_INFO "Invalid segreg in modrm byte 0x%02x\n",
-                                       c->modrm);
-                       goto cannot_emulate;
+               if (c->modrm_reg == VCPU_SREG_CS ||
+                   c->modrm_reg > VCPU_SREG_GS) {
+                       kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
+                       goto done;
                }
 
-               if (err < 0)
-                       goto cannot_emulate;
+               if (c->modrm_reg == VCPU_SREG_SS)
+                       toggle_interruptibility(ctxt, KVM_X86_SHADOW_INT_MOV_SS);
+
+               rc = kvm_load_segment_descriptor(ctxt->vcpu, sel, c->modrm_reg);
 
                c->dst.type = OP_NONE;  /* Disable writeback. */
                break;
        }
        case 0x8f:              /* pop (sole member of Grp1a) */
                rc = emulate_grp1a(ctxt, ops);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0x90: /* nop / xchg r8,rax */
@@ -2283,7 +2295,7 @@ special_insn:
                break;
        case 0xcb:              /* ret far */
                rc = emulate_ret_far(ctxt, ops);
-               if (rc)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0xd0 ... 0xd1:     /* Grp2 */
@@ -2314,11 +2326,10 @@ special_insn:
        case 0xe9: /* jmp rel */
                goto jmp;
        case 0xea: /* jmp far */
-               if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val, 9,
-                                       VCPU_SREG_CS) < 0) {
-                       DPRINTF("jmp far: Failed to load CS descriptor\n");
-                       goto cannot_emulate;
-               }
+       jump_far:
+               if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val,
+                                               VCPU_SREG_CS))
+                       goto done;
 
                c->eip = c->src.val;
                break;
@@ -2359,7 +2370,7 @@ special_insn:
                break;
        case 0xf6 ... 0xf7:     /* Grp3 */
                rc = emulate_grp3(ctxt, ops);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0xf8: /* clc */
@@ -2367,7 +2378,7 @@ special_insn:
                c->dst.type = OP_NONE;  /* Disable writeback. */
                break;
        case 0xfa: /* cli */
-               if (emulator_bad_iopl(ctxt))
+               if (emulator_bad_iopl(ctxt, ops))
                        kvm_inject_gp(ctxt->vcpu, 0);
                else {
                        ctxt->eflags &= ~X86_EFLAGS_IF;
@@ -2375,10 +2386,10 @@ special_insn:
                }
                break;
        case 0xfb: /* sti */
-               if (emulator_bad_iopl(ctxt))
+               if (emulator_bad_iopl(ctxt, ops))
                        kvm_inject_gp(ctxt->vcpu, 0);
                else {
-                       toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
+                       toggle_interruptibility(ctxt, KVM_X86_SHADOW_INT_STI);
                        ctxt->eflags |= X86_EFLAGS_IF;
                        c->dst.type = OP_NONE;  /* Disable writeback. */
                }
@@ -2391,16 +2402,21 @@ special_insn:
                ctxt->eflags |= EFLG_DF;
                c->dst.type = OP_NONE;  /* Disable writeback. */
                break;
-       case 0xfe ... 0xff:     /* Grp4/Grp5 */
+       case 0xfe: /* Grp4 */
+       grp45:
                rc = emulate_grp45(ctxt, ops);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
+       case 0xff: /* Grp5 */
+               if (c->modrm_reg == 5)
+                       goto jump_far;
+               goto grp45;
        }
 
 writeback:
        rc = writeback(ctxt, ops);
-       if (rc != 0)
+       if (rc != X86EMUL_CONTINUE)
                goto done;
 
        /* Commit shadow register state. */
@@ -2426,18 +2442,18 @@ twobyte_insn:
                                goto cannot_emulate;
 
                        rc = kvm_fix_hypercall(ctxt->vcpu);
-                       if (rc)
+                       if (rc != X86EMUL_CONTINUE)
                                goto done;
 
                        /* Let the processor re-execute the fixed hypercall */
-                       c->eip = kvm_rip_read(ctxt->vcpu);
+                       c->eip = ctxt->eip;
                        /* Disable writeback. */
                        c->dst.type = OP_NONE;
                        break;
                case 2: /* lgdt */
                        rc = read_descriptor(ctxt, ops, c->src.ptr,
                                             &size, &address, c->op_bytes);
-                       if (rc)
+                       if (rc != X86EMUL_CONTINUE)
                                goto done;
                        realmode_lgdt(ctxt->vcpu, size, address);
                        /* Disable writeback. */
@@ -2448,7 +2464,7 @@ twobyte_insn:
                                switch (c->modrm_rm) {
                                case 1:
                                        rc = kvm_fix_hypercall(ctxt->vcpu);
-                                       if (rc)
+                                       if (rc != X86EMUL_CONTINUE)
                                                goto done;
                                        break;
                                default:
@@ -2458,7 +2474,7 @@ twobyte_insn:
                                rc = read_descriptor(ctxt, ops, c->src.ptr,
                                                     &size, &address,
                                                     c->op_bytes);
-                               if (rc)
+                               if (rc != X86EMUL_CONTINUE)
                                        goto done;
                                realmode_lidt(ctxt->vcpu, size, address);
                        }
@@ -2467,13 +2483,16 @@ twobyte_insn:
                        break;
                case 4: /* smsw */
                        c->dst.bytes = 2;
-                       c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
+                       c->dst.val = ops->get_cr(0, ctxt->vcpu);
                        break;
                case 6: /* lmsw */
-                       realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
-                                     &ctxt->eflags);
+                       ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
+                                   (c->src.val & 0x0f), ctxt->vcpu);
                        c->dst.type = OP_NONE;
                        break;
+               case 5: /* not defined */
+                       kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
+                       goto done;
                case 7: /* invlpg*/
                        emulate_invlpg(ctxt->vcpu, memop);
                        /* Disable writeback. */
@@ -2484,8 +2503,9 @@ twobyte_insn:
                }
                break;
        case 0x05:              /* syscall */
-               if (emulate_syscall(ctxt) == -1)
-                       goto cannot_emulate;
+               rc = emulate_syscall(ctxt);
+               if (rc != X86EMUL_CONTINUE)
+                       goto done;
                else
                        goto writeback;
                break;
@@ -2502,52 +2522,47 @@ twobyte_insn:
        case 0x20: /* mov cr, reg */
                if (c->modrm_mod != 3)
                        goto cannot_emulate;
-               c->regs[c->modrm_rm] =
-                               realmode_get_cr(ctxt->vcpu, c->modrm_reg);
+               c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
                c->dst.type = OP_NONE;  /* no writeback */
                break;
        case 0x21: /* mov from dr to reg */
                if (c->modrm_mod != 3)
                        goto cannot_emulate;
-               rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
-               if (rc)
+               if (emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]))
                        goto cannot_emulate;
+               rc = X86EMUL_CONTINUE;
                c->dst.type = OP_NONE;  /* no writeback */
                break;
        case 0x22: /* mov reg, cr */
                if (c->modrm_mod != 3)
                        goto cannot_emulate;
-               realmode_set_cr(ctxt->vcpu,
-                               c->modrm_reg, c->modrm_val, &ctxt->eflags);
+               ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu);
                c->dst.type = OP_NONE;
                break;
        case 0x23: /* mov from reg to dr */
                if (c->modrm_mod != 3)
                        goto cannot_emulate;
-               rc = emulator_set_dr(ctxt, c->modrm_reg,
-                                    c->regs[c->modrm_rm]);
-               if (rc)
+               if (emulator_set_dr(ctxt, c->modrm_reg, c->regs[c->modrm_rm]))
                        goto cannot_emulate;
+               rc = X86EMUL_CONTINUE;
                c->dst.type = OP_NONE;  /* no writeback */
                break;
        case 0x30:
                /* wrmsr */
                msr_data = (u32)c->regs[VCPU_REGS_RAX]
                        | ((u64)c->regs[VCPU_REGS_RDX] << 32);
-               rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
-               if (rc) {
+               if (kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
                        kvm_inject_gp(ctxt->vcpu, 0);
-                       c->eip = kvm_rip_read(ctxt->vcpu);
+                       c->eip = ctxt->eip;
                }
                rc = X86EMUL_CONTINUE;
                c->dst.type = OP_NONE;
                break;
        case 0x32:
                /* rdmsr */
-               rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
-               if (rc) {
+               if (kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
                        kvm_inject_gp(ctxt->vcpu, 0);
-                       c->eip = kvm_rip_read(ctxt->vcpu);
+                       c->eip = ctxt->eip;
                } else {
                        c->regs[VCPU_REGS_RAX] = (u32)msr_data;
                        c->regs[VCPU_REGS_RDX] = msr_data >> 32;
@@ -2556,14 +2571,16 @@ twobyte_insn:
                c->dst.type = OP_NONE;
                break;
        case 0x34:              /* sysenter */
-               if (emulate_sysenter(ctxt) == -1)
-                       goto cannot_emulate;
+               rc = emulate_sysenter(ctxt);
+               if (rc != X86EMUL_CONTINUE)
+                       goto done;
                else
                        goto writeback;
                break;
        case 0x35:              /* sysexit */
-               if (emulate_sysexit(ctxt) == -1)
-                       goto cannot_emulate;
+               rc = emulate_sysexit(ctxt);
+               if (rc != X86EMUL_CONTINUE)
+                       goto done;
                else
                        goto writeback;
                break;
@@ -2582,7 +2599,7 @@ twobyte_insn:
                break;
        case 0xa1:       /* pop fs */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0xa3:
@@ -2601,7 +2618,7 @@ twobyte_insn:
                break;
        case 0xa9:      /* pop gs */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                break;
        case 0xab:
@@ -2674,7 +2691,7 @@ twobyte_insn:
                break;
        case 0xc7:              /* Grp9 (cmpxchg8b) */
                rc = emulate_grp9(ctxt, ops, memop);
-               if (rc != 0)
+               if (rc != X86EMUL_CONTINUE)
                        goto done;
                c->dst.type = OP_NONE;
                break;