KVM: x86: Push potential exception error code on task switches
[safe/jmp/linux-2.6] / arch / x86 / kvm / x86.c
index f69854c..58a295c 100644 (file)
@@ -562,6 +562,80 @@ unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_get_cr8);
 
+int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
+{
+       switch (dr) {
+       case 0 ... 3:
+               vcpu->arch.db[dr] = val;
+               if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
+                       vcpu->arch.eff_db[dr] = val;
+               break;
+       case 4:
+               if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) {
+                       kvm_queue_exception(vcpu, UD_VECTOR);
+                       return 1;
+               }
+               /* fall through */
+       case 6:
+               if (val & 0xffffffff00000000ULL) {
+                       kvm_inject_gp(vcpu, 0);
+                       return 1;
+               }
+               vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
+               break;
+       case 5:
+               if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) {
+                       kvm_queue_exception(vcpu, UD_VECTOR);
+                       return 1;
+               }
+               /* fall through */
+       default: /* 7 */
+               if (val & 0xffffffff00000000ULL) {
+                       kvm_inject_gp(vcpu, 0);
+                       return 1;
+               }
+               vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
+               if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
+                       kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7);
+                       vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK);
+               }
+               break;
+       }
+
+       return 0;
+}
+EXPORT_SYMBOL_GPL(kvm_set_dr);
+
+int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
+{
+       switch (dr) {
+       case 0 ... 3:
+               *val = vcpu->arch.db[dr];
+               break;
+       case 4:
+               if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) {
+                       kvm_queue_exception(vcpu, UD_VECTOR);
+                       return 1;
+               }
+               /* fall through */
+       case 6:
+               *val = vcpu->arch.dr6;
+               break;
+       case 5:
+               if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) {
+                       kvm_queue_exception(vcpu, UD_VECTOR);
+                       return 1;
+               }
+               /* fall through */
+       default: /* 7 */
+               *val = vcpu->arch.dr7;
+               break;
+       }
+
+       return 0;
+}
+EXPORT_SYMBOL_GPL(kvm_get_dr);
+
 static inline u32 bit(int bitno)
 {
        return 1 << (bitno & 31);
@@ -3150,18 +3224,17 @@ static int kvm_read_guest_virt_system(gva_t addr, void *val, unsigned int bytes,
        return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, error);
 }
 
-static int kvm_write_guest_virt_helper(gva_t addr, void *val,
+static int kvm_write_guest_virt_system(gva_t addr, void *val,
                                       unsigned int bytes,
-                                      struct kvm_vcpu *vcpu, u32 access,
+                                      struct kvm_vcpu *vcpu,
                                       u32 *error)
 {
        void *data = val;
        int r = X86EMUL_CONTINUE;
 
-       access |= PFERR_WRITE_MASK;
-
        while (bytes) {
-               gpa_t gpa =  vcpu->arch.mmu.gva_to_gpa(vcpu, addr, access, error);
+               gpa_t gpa =  vcpu->arch.mmu.gva_to_gpa(vcpu, addr,
+                                                      PFERR_WRITE_MASK, error);
                unsigned offset = addr & (PAGE_SIZE-1);
                unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
                int ret;
@@ -3184,20 +3257,6 @@ out:
        return r;
 }
 
-static int kvm_write_guest_virt(gva_t addr, void *val, unsigned int bytes,
-                               struct kvm_vcpu *vcpu, u32 *error)
-{
-       u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
-       return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, access, error);
-}
-
-static int kvm_write_guest_virt_system(gva_t addr, void *val,
-                                      unsigned int bytes,
-                                      struct kvm_vcpu *vcpu, u32 *error)
-{
-       return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, 0, error);
-}
-
 static int emulator_read_emulated(unsigned long addr,
                                  void *val,
                                  unsigned int bytes,
@@ -3263,8 +3322,7 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 static int emulator_write_emulated_onepage(unsigned long addr,
                                           const void *val,
                                           unsigned int bytes,
-                                          struct kvm_vcpu *vcpu,
-                                          bool mmu_only)
+                                          struct kvm_vcpu *vcpu)
 {
        gpa_t                 gpa;
        u32 error_code;
@@ -3280,10 +3338,6 @@ static int emulator_write_emulated_onepage(unsigned long addr,
        if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
                goto mmio;
 
-       if (mmu_only) {
-               kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
-               return X86EMUL_CONTINUE;
-       }
        if (emulator_write_phys(vcpu, gpa, val, bytes))
                return X86EMUL_CONTINUE;
 
@@ -3304,35 +3358,24 @@ mmio:
        return X86EMUL_CONTINUE;
 }
 
-int __emulator_write_emulated(unsigned long addr,
-                                  const void *val,
-                                  unsigned int bytes,
-                                  struct kvm_vcpu *vcpu,
-                                  bool mmu_only)
+int emulator_write_emulated(unsigned long addr,
+                           const void *val,
+                           unsigned int bytes,
+                           struct kvm_vcpu *vcpu)
 {
        /* Crossing a page boundary? */
        if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
                int rc, now;
 
                now = -addr & ~PAGE_MASK;
-               rc = emulator_write_emulated_onepage(addr, val, now, vcpu,
-                                                    mmu_only);
+               rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
                if (rc != X86EMUL_CONTINUE)
                        return rc;
                addr += now;
                val += now;
                bytes -= now;
        }
-       return emulator_write_emulated_onepage(addr, val, bytes, vcpu,
-                                              mmu_only);
-}
-
-int emulator_write_emulated(unsigned long addr,
-                                  const void *val,
-                                  unsigned int bytes,
-                                  struct kvm_vcpu *vcpu)
-{
-       return __emulator_write_emulated(addr, val, bytes, vcpu, false);
+       return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
 }
 EXPORT_SYMBOL_GPL(emulator_write_emulated);
 
@@ -3343,7 +3386,7 @@ EXPORT_SYMBOL_GPL(emulator_write_emulated);
 #  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
 #else
 #  define CMPXCHG64(ptr, old, new) \
-       (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u *)(new)) == *(u64 *)(old))
+       (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
 #endif
 
 static int emulator_cmpxchg_emulated(unsigned long addr,
@@ -3396,7 +3439,9 @@ static int emulator_cmpxchg_emulated(unsigned long addr,
        if (!exchanged)
                return X86EMUL_CMPXCHG_FAILED;
 
-       return __emulator_write_emulated(addr, new, bytes, vcpu, true);
+       kvm_mmu_pte_write(vcpu, gpa, new, bytes, 1);
+
+       return X86EMUL_CONTINUE;
 
 emul_write:
        printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
@@ -3404,6 +3449,80 @@ emul_write:
        return emulator_write_emulated(addr, new, bytes, vcpu);
 }
 
+static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
+{
+       /* TODO: String I/O for in kernel device */
+       int r;
+
+       if (vcpu->arch.pio.in)
+               r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
+                                   vcpu->arch.pio.size, pd);
+       else
+               r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
+                                    vcpu->arch.pio.port, vcpu->arch.pio.size,
+                                    pd);
+       return r;
+}
+
+
+static int emulator_pio_in_emulated(int size, unsigned short port, void *val,
+                            unsigned int count, struct kvm_vcpu *vcpu)
+{
+       if (vcpu->arch.pio.count)
+               goto data_avail;
+
+       trace_kvm_pio(1, port, size, 1);
+
+       vcpu->arch.pio.port = port;
+       vcpu->arch.pio.in = 1;
+       vcpu->arch.pio.count  = count;
+       vcpu->arch.pio.size = size;
+
+       if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
+       data_avail:
+               memcpy(val, vcpu->arch.pio_data, size * count);
+               vcpu->arch.pio.count = 0;
+               return 1;
+       }
+
+       vcpu->run->exit_reason = KVM_EXIT_IO;
+       vcpu->run->io.direction = KVM_EXIT_IO_IN;
+       vcpu->run->io.size = size;
+       vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
+       vcpu->run->io.count = count;
+       vcpu->run->io.port = port;
+
+       return 0;
+}
+
+static int emulator_pio_out_emulated(int size, unsigned short port,
+                             const void *val, unsigned int count,
+                             struct kvm_vcpu *vcpu)
+{
+       trace_kvm_pio(0, port, size, 1);
+
+       vcpu->arch.pio.port = port;
+       vcpu->arch.pio.in = 0;
+       vcpu->arch.pio.count = count;
+       vcpu->arch.pio.size = size;
+
+       memcpy(vcpu->arch.pio_data, val, size * count);
+
+       if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
+               vcpu->arch.pio.count = 0;
+               return 1;
+       }
+
+       vcpu->run->exit_reason = KVM_EXIT_IO;
+       vcpu->run->io.direction = KVM_EXIT_IO_OUT;
+       vcpu->run->io.size = size;
+       vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
+       vcpu->run->io.count = count;
+       vcpu->run->io.port = port;
+
+       return 0;
+}
+
 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
 {
        return kvm_x86_ops->get_segment_base(vcpu, seg);
@@ -3424,14 +3543,14 @@ int emulate_clts(struct kvm_vcpu *vcpu)
 
 int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
 {
-       return kvm_x86_ops->get_dr(ctxt->vcpu, dr, dest);
+       return kvm_get_dr(ctxt->vcpu, dr, dest);
 }
 
 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
 {
        unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
 
-       return kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask);
+       return kvm_set_dr(ctxt->vcpu, dr, value & mask);
 }
 
 void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
@@ -3590,6 +3709,11 @@ static void emulator_set_segment_selector(u16 sel, int seg,
        kvm_set_segment(vcpu, &kvm_seg, seg);
 }
 
+static void emulator_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
+{
+       kvm_x86_ops->set_rflags(vcpu, rflags);
+}
+
 static struct x86_emulate_ops emulate_ops = {
        .read_std            = kvm_read_guest_virt_system,
        .write_std           = kvm_write_guest_virt_system,
@@ -3597,6 +3721,8 @@ static struct x86_emulate_ops emulate_ops = {
        .read_emulated       = emulator_read_emulated,
        .write_emulated      = emulator_write_emulated,
        .cmpxchg_emulated    = emulator_cmpxchg_emulated,
+       .pio_in_emulated     = emulator_pio_in_emulated,
+       .pio_out_emulated    = emulator_pio_out_emulated,
        .get_cached_descriptor = emulator_get_cached_descriptor,
        .set_cached_descriptor = emulator_set_cached_descriptor,
        .get_segment_selector = emulator_get_segment_selector,
@@ -3605,6 +3731,7 @@ static struct x86_emulate_ops emulate_ops = {
        .get_cr              = emulator_get_cr,
        .set_cr              = emulator_set_cr,
        .cpl                 = emulator_get_cpl,
+       .set_rflags          = emulator_set_rflags,
 };
 
 static void cache_all_regs(struct kvm_vcpu *vcpu)
@@ -3635,7 +3762,6 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
        cache_all_regs(vcpu);
 
        vcpu->mmio_is_write = 0;
-       vcpu->arch.pio.string = 0;
 
        if (!(emulation_type & EMULTYPE_NO_DECODE)) {
                int cs_db, cs_l;
@@ -3652,6 +3778,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
                        ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
 
                r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
+               trace_kvm_emulate_insn_start(vcpu);
 
                /* Only allow emulation of specific instructions on #UD
                 * (namely VMMCALL, sysenter, sysexit, syscall)*/
@@ -3684,6 +3811,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
                ++vcpu->stat.insn_emulation;
                if (r)  {
                        ++vcpu->stat.insn_emulation_fail;
+                       trace_kvm_emulate_insn_failed(vcpu);
                        if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
                                return EMULATE_DONE;
                        return EMULATE_FAIL;
@@ -3695,14 +3823,18 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
                return EMULATE_DONE;
        }
 
+restart:
        r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
        shadow_mask = vcpu->arch.emulate_ctxt.interruptibility;
 
        if (r == 0)
                kvm_x86_ops->set_interrupt_shadow(vcpu, shadow_mask);
 
-       if (vcpu->arch.pio.string)
+       if (vcpu->arch.pio.count) {
+               if (!vcpu->arch.pio.in)
+                       vcpu->arch.pio.count = 0;
                return EMULATE_DO_MMIO;
+       }
 
        if (r || vcpu->mmio_is_write) {
                run->exit_reason = KVM_EXIT_MMIO;
@@ -3714,222 +3846,41 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
 
        if (r) {
                if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
-                       return EMULATE_DONE;
+                       goto done;
                if (!vcpu->mmio_needed) {
+                       ++vcpu->stat.insn_emulation_fail;
+                       trace_kvm_emulate_insn_failed(vcpu);
                        kvm_report_emulation_failure(vcpu, "mmio");
                        return EMULATE_FAIL;
                }
                return EMULATE_DO_MMIO;
        }
 
-       kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
-
        if (vcpu->mmio_is_write) {
                vcpu->mmio_needed = 0;
                return EMULATE_DO_MMIO;
        }
 
-       return EMULATE_DONE;
-}
-EXPORT_SYMBOL_GPL(emulate_instruction);
-
-static int pio_copy_data(struct kvm_vcpu *vcpu)
-{
-       void *p = vcpu->arch.pio_data;
-       gva_t q = vcpu->arch.pio.guest_gva;
-       unsigned bytes;
-       int ret;
-       u32 error_code;
-
-       bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count;
-       if (vcpu->arch.pio.in)
-               ret = kvm_write_guest_virt(q, p, bytes, vcpu, &error_code);
-       else
-               ret = kvm_read_guest_virt(q, p, bytes, vcpu, &error_code);
-
-       if (ret == X86EMUL_PROPAGATE_FAULT)
-               kvm_inject_page_fault(vcpu, q, error_code);
-
-       return ret;
-}
-
-int complete_pio(struct kvm_vcpu *vcpu)
-{
-       struct kvm_pio_request *io = &vcpu->arch.pio;
-       long delta;
-       int r;
-       unsigned long val;
-
-       if (!io->string) {
-               if (io->in) {
-                       val = kvm_register_read(vcpu, VCPU_REGS_RAX);
-                       memcpy(&val, vcpu->arch.pio_data, io->size);
-                       kvm_register_write(vcpu, VCPU_REGS_RAX, val);
-               }
-       } else {
-               if (io->in) {
-                       r = pio_copy_data(vcpu);
-                       if (r)
-                               goto out;
-               }
-
-               delta = 1;
-               if (io->rep) {
-                       delta *= io->cur_count;
-                       /*
-                        * The size of the register should really depend on
-                        * current address size.
-                        */
-                       val = kvm_register_read(vcpu, VCPU_REGS_RCX);
-                       val -= delta;
-                       kvm_register_write(vcpu, VCPU_REGS_RCX, val);
-               }
-               if (io->down)
-                       delta = -delta;
-               delta *= io->size;
-               if (io->in) {
-                       val = kvm_register_read(vcpu, VCPU_REGS_RDI);
-                       val += delta;
-                       kvm_register_write(vcpu, VCPU_REGS_RDI, val);
-               } else {
-                       val = kvm_register_read(vcpu, VCPU_REGS_RSI);
-                       val += delta;
-                       kvm_register_write(vcpu, VCPU_REGS_RSI, val);
-               }
-       }
-out:
-       io->count -= io->cur_count;
-       io->cur_count = 0;
-
-       return 0;
-}
-
-static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
-{
-       /* TODO: String I/O for in kernel device */
-       int r;
-
-       if (vcpu->arch.pio.in)
-               r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
-                                   vcpu->arch.pio.size, pd);
-       else
-               r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
-                                    vcpu->arch.pio.port, vcpu->arch.pio.size,
-                                    pd);
-       return r;
-}
-
-static int pio_string_write(struct kvm_vcpu *vcpu)
-{
-       struct kvm_pio_request *io = &vcpu->arch.pio;
-       void *pd = vcpu->arch.pio_data;
-       int i, r = 0;
-
-       for (i = 0; i < io->cur_count; i++) {
-               if (kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
-                                    io->port, io->size, pd)) {
-                       r = -EOPNOTSUPP;
-                       break;
-               }
-               pd += io->size;
-       }
-       return r;
-}
-
-int kvm_emulate_pio(struct kvm_vcpu *vcpu, int in, int size, unsigned port)
-{
-       unsigned long val;
-
-       trace_kvm_pio(!in, port, size, 1);
-
-       vcpu->run->exit_reason = KVM_EXIT_IO;
-       vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
-       vcpu->run->io.size = vcpu->arch.pio.size = size;
-       vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
-       vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1;
-       vcpu->run->io.port = vcpu->arch.pio.port = port;
-       vcpu->arch.pio.in = in;
-       vcpu->arch.pio.string = 0;
-       vcpu->arch.pio.down = 0;
-       vcpu->arch.pio.rep = 0;
+done:
+       if (vcpu->arch.exception.pending)
+               vcpu->arch.emulate_ctxt.restart = false;
 
-       if (!vcpu->arch.pio.in) {
-               val = kvm_register_read(vcpu, VCPU_REGS_RAX);
-               memcpy(vcpu->arch.pio_data, &val, 4);
-       }
+       if (vcpu->arch.emulate_ctxt.restart)
+               goto restart;
 
-       if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
-               complete_pio(vcpu);
-               return 1;
-       }
-       return 0;
+       return EMULATE_DONE;
 }
-EXPORT_SYMBOL_GPL(kvm_emulate_pio);
+EXPORT_SYMBOL_GPL(emulate_instruction);
 
-int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, int in,
-                 int size, unsigned long count, int down,
-                 gva_t address, int rep, unsigned port)
+int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
 {
-       unsigned now, in_page;
-       int ret = 0;
-
-       trace_kvm_pio(!in, port, size, count);
-
-       vcpu->run->exit_reason = KVM_EXIT_IO;
-       vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
-       vcpu->run->io.size = vcpu->arch.pio.size = size;
-       vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
-       vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count;
-       vcpu->run->io.port = vcpu->arch.pio.port = port;
-       vcpu->arch.pio.in = in;
-       vcpu->arch.pio.string = 1;
-       vcpu->arch.pio.down = down;
-       vcpu->arch.pio.rep = rep;
-
-       if (!count) {
-               kvm_x86_ops->skip_emulated_instruction(vcpu);
-               return 1;
-       }
-
-       if (!down)
-               in_page = PAGE_SIZE - offset_in_page(address);
-       else
-               in_page = offset_in_page(address) + size;
-       now = min(count, (unsigned long)in_page / size);
-       if (!now)
-               now = 1;
-       if (down) {
-               /*
-                * String I/O in reverse.  Yuck.  Kill the guest, fix later.
-                */
-               pr_unimpl(vcpu, "guest string pio down\n");
-               kvm_inject_gp(vcpu, 0);
-               return 1;
-       }
-       vcpu->run->io.count = now;
-       vcpu->arch.pio.cur_count = now;
-
-       if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count)
-               kvm_x86_ops->skip_emulated_instruction(vcpu);
-
-       vcpu->arch.pio.guest_gva = address;
-
-       if (!vcpu->arch.pio.in) {
-               /* string PIO write */
-               ret = pio_copy_data(vcpu);
-               if (ret == X86EMUL_PROPAGATE_FAULT)
-                       return 1;
-               if (ret == 0 && !pio_string_write(vcpu)) {
-                       complete_pio(vcpu);
-                       if (vcpu->arch.pio.count == 0)
-                               ret = 1;
-               }
-       }
-       /* no string PIO read support yet */
-
+       unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
+       int ret = emulator_pio_out_emulated(size, port, &val, 1, vcpu);
+       /* do not return to emulator after return from userspace */
+       vcpu->arch.pio.count = 0;
        return ret;
 }
-EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
+EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
 
 static void bounce_off(void *info)
 {
@@ -4200,7 +4151,7 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
 
        kvm_x86_ops->patch_hypercall(vcpu, instruction);
 
-       return __emulator_write_emulated(rip, instruction, 3, vcpu, false);
+       return emulator_write_emulated(rip, instruction, 3, vcpu);
 }
 
 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
@@ -4280,9 +4231,13 @@ int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
 {
        struct kvm_cpuid_entry2 *best;
 
+       best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
+       if (!best || best->eax < 0x80000008)
+               goto not_found;
        best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
        if (best)
                return best->eax & 0xff;
+not_found:
        return 36;
 }
 
@@ -4659,26 +4614,17 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
        if (!irqchip_in_kernel(vcpu->kvm))
                kvm_set_cr8(vcpu, kvm_run->cr8);
 
-       if (vcpu->arch.pio.cur_count) {
-               vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
-               r = complete_pio(vcpu);
-               srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
-               if (r)
-                       goto out;
-       }
-       if (vcpu->mmio_needed) {
-               memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
-               vcpu->mmio_read_completed = 1;
-               vcpu->mmio_needed = 0;
-
+       if (vcpu->arch.pio.count || vcpu->mmio_needed ||
+           vcpu->arch.emulate_ctxt.restart) {
+               if (vcpu->mmio_needed) {
+                       memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
+                       vcpu->mmio_read_completed = 1;
+                       vcpu->mmio_needed = 0;
+               }
                vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
-               r = emulate_instruction(vcpu, vcpu->arch.mmio_fault_cr2, 0,
-                                       EMULTYPE_NO_DECODE);
+               r = emulate_instruction(vcpu, 0, 0, EMULTYPE_NO_DECODE);
                srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
                if (r == EMULATE_DO_MMIO) {
-                       /*
-                        * Read-modify-write.  Back to userspace.
-                        */
                        r = 0;
                        goto out;
                }
@@ -4832,7 +4778,8 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
        return 0;
 }
 
-int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
+int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
+                   bool has_error_code, u32 error_code)
 {
        int cs_db, cs_l, ret;
        cache_all_regs(vcpu);
@@ -4850,7 +4797,8 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
                ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
 
        ret = emulator_task_switch(&vcpu->arch.emulate_ctxt, &emulate_ops,
-                                  tss_selector, reason);
+                                  tss_selector, reason, has_error_code,
+                                  error_code);
 
        if (ret == X86EMUL_CONTINUE)
                kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);