SAFE public projects git trees. - safe/jmp/linux-2.6/blob - arch/x86/kvm/x86.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * derived from drivers/kvm/kvm_main.c
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright (C) 2008 Qumranet, Inc.
   8  * Copyright IBM Corporation, 2008
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *   Amit Shah    <amit.shah@qumranet.com>
  14  *   Ben-Ami Yassour <benami@il.ibm.com>
  15  *
  16  * This work is licensed under the terms of the GNU GPL, version 2.  See
  17  * the COPYING file in the top-level directory.
  18  *
  19  */
  20
  21 #include <linux/kvm_host.h>
  22 #include "irq.h"
  23 #include "mmu.h"
  24 #include "i8254.h"
  25 #include "tss.h"
  26 #include "kvm_cache_regs.h"
  27 #include "x86.h"
  28
  29 #include <linux/clocksource.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/kvm.h>
  32 #include <linux/fs.h>
  33 #include <linux/vmalloc.h>
  34 #include <linux/module.h>
  35 #include <linux/mman.h>
  36 #include <linux/highmem.h>
  37 #include <linux/iommu.h>
  38 #include <linux/intel-iommu.h>
  39 #include <linux/cpufreq.h>
  40
  41 #include <asm/uaccess.h>
  42 #include <asm/msr.h>
  43 #include <asm/desc.h>
  44 #include <asm/mtrr.h>
  45 #include <asm/mce.h>
  46
  47 #define MAX_IO_MSRS 256
  48 #define CR0_RESERVED_BITS                                               \
  49         (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  50                           | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  51                           | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  52 #define CR4_RESERVED_BITS                                               \
  53         (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  54                           | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
  55                           | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR  \
  56                           | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  57
  58 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  59
  60 #define KVM_MAX_MCE_BANKS 32
  61 #define KVM_MCE_CAP_SUPPORTED MCG_CTL_P
  62
  63 /* EFER defaults:
  64  * - enable syscall per default because its emulated by KVM
  65  * - enable LME and LMA per default on 64 bit KVM
  66  */
  67 #ifdef CONFIG_X86_64
  68 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffafeULL;
  69 #else
  70 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
  71 #endif
  72
  73 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  74 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  75
  76 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  77                                     struct kvm_cpuid_entry2 __user *entries);
  78 struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
  79                                               u32 function, u32 index);
  80
  81 struct kvm_x86_ops *kvm_x86_ops;
  82 EXPORT_SYMBOL_GPL(kvm_x86_ops);
  83
  84 struct kvm_stats_debugfs_item debugfs_entries[] = {
  85         { "pf_fixed", VCPU_STAT(pf_fixed) },
  86         { "pf_guest", VCPU_STAT(pf_guest) },
  87         { "tlb_flush", VCPU_STAT(tlb_flush) },
  88         { "invlpg", VCPU_STAT(invlpg) },
  89         { "exits", VCPU_STAT(exits) },
  90         { "io_exits", VCPU_STAT(io_exits) },
  91         { "mmio_exits", VCPU_STAT(mmio_exits) },
  92         { "signal_exits", VCPU_STAT(signal_exits) },
  93         { "irq_window", VCPU_STAT(irq_window_exits) },
  94         { "nmi_window", VCPU_STAT(nmi_window_exits) },
  95         { "halt_exits", VCPU_STAT(halt_exits) },
  96         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  97         { "hypercalls", VCPU_STAT(hypercalls) },
  98         { "request_irq", VCPU_STAT(request_irq_exits) },
  99         { "irq_exits", VCPU_STAT(irq_exits) },
 100         { "host_state_reload", VCPU_STAT(host_state_reload) },
 101         { "efer_reload", VCPU_STAT(efer_reload) },
 102         { "fpu_reload", VCPU_STAT(fpu_reload) },
 103         { "insn_emulation", VCPU_STAT(insn_emulation) },
 104         { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
 105         { "irq_injections", VCPU_STAT(irq_injections) },
 106         { "nmi_injections", VCPU_STAT(nmi_injections) },
 107         { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
 108         { "mmu_pte_write", VM_STAT(mmu_pte_write) },
 109         { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
 110         { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
 111         { "mmu_flooded", VM_STAT(mmu_flooded) },
 112         { "mmu_recycled", VM_STAT(mmu_recycled) },
 113         { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
 114         { "mmu_unsync", VM_STAT(mmu_unsync) },
 115         { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
 116         { "largepages", VM_STAT(lpages) },
 117         { NULL }
 118 };
 119
 120 unsigned long segment_base(u16 selector)
 121 {
 122         struct descriptor_table gdt;
 123         struct desc_struct *d;
 124         unsigned long table_base;
 125         unsigned long v;
 126
 127         if (selector == 0)
 128                 return 0;
 129
 130         asm("sgdt %0" : "=m"(gdt));
 131         table_base = gdt.base;
 132
 133         if (selector & 4) {           /* from ldt */
 134                 u16 ldt_selector;
 135
 136                 asm("sldt %0" : "=g"(ldt_selector));
 137                 table_base = segment_base(ldt_selector);
 138         }
 139         d = (struct desc_struct *)(table_base + (selector & ~7));
 140         v = d->base0 | ((unsigned long)d->base1 << 16) |
 141                 ((unsigned long)d->base2 << 24);
 142 #ifdef CONFIG_X86_64
 143         if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
 144                 v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
 145 #endif
 146         return v;
 147 }
 148 EXPORT_SYMBOL_GPL(segment_base);
 149
 150 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
 151 {
 152         if (irqchip_in_kernel(vcpu->kvm))
 153                 return vcpu->arch.apic_base;
 154         else
 155                 return vcpu->arch.apic_base;
 156 }
 157 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
 158
 159 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
 160 {
 161         /* TODO: reserve bits check */
 162         if (irqchip_in_kernel(vcpu->kvm))
 163                 kvm_lapic_set_base(vcpu, data);
 164         else
 165                 vcpu->arch.apic_base = data;
 166 }
 167 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
 168
 169 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 170 {
 171         WARN_ON(vcpu->arch.exception.pending);
 172         vcpu->arch.exception.pending = true;
 173         vcpu->arch.exception.has_error_code = false;
 174         vcpu->arch.exception.nr = nr;
 175 }
 176 EXPORT_SYMBOL_GPL(kvm_queue_exception);
 177
 178 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
 179                            u32 error_code)
 180 {
 181         ++vcpu->stat.pf_guest;
 182
 183         if (vcpu->arch.exception.pending) {
 184                 if (vcpu->arch.exception.nr == PF_VECTOR) {
 185                         printk(KERN_DEBUG "kvm: inject_page_fault:"
 186                                         " double fault 0x%lx\n", addr);
 187                         vcpu->arch.exception.nr = DF_VECTOR;
 188                         vcpu->arch.exception.error_code = 0;
 189                 } else if (vcpu->arch.exception.nr == DF_VECTOR) {
 190                         /* triple fault -> shutdown */
 191                         set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
 192                 }
 193                 return;
 194         }
 195         vcpu->arch.cr2 = addr;
 196         kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
 197 }
 198
 199 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
 200 {
 201         vcpu->arch.nmi_pending = 1;
 202 }
 203 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
 204
 205 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 206 {
 207         WARN_ON(vcpu->arch.exception.pending);
 208         vcpu->arch.exception.pending = true;
 209         vcpu->arch.exception.has_error_code = true;
 210         vcpu->arch.exception.nr = nr;
 211         vcpu->arch.exception.error_code = error_code;
 212 }
 213 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
 214
 215 static void __queue_exception(struct kvm_vcpu *vcpu)
 216 {
 217         kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
 218                                      vcpu->arch.exception.has_error_code,
 219                                      vcpu->arch.exception.error_code);
 220 }
 221
 222 /*
 223  * Load the pae pdptrs.  Return true is they are all valid.
 224  */
 225 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 226 {
 227         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 228         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 229         int i;
 230         int ret;
 231         u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 232
 233         ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
 234                                   offset * sizeof(u64), sizeof(pdpte));
 235         if (ret < 0) {
 236                 ret = 0;
 237                 goto out;
 238         }
 239         for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
 240                 if (is_present_gpte(pdpte[i]) &&
 241                     (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
 242                         ret = 0;
 243                         goto out;
 244                 }
 245         }
 246         ret = 1;
 247
 248         memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
 249         __set_bit(VCPU_EXREG_PDPTR,
 250                   (unsigned long *)&vcpu->arch.regs_avail);
 251         __set_bit(VCPU_EXREG_PDPTR,
 252                   (unsigned long *)&vcpu->arch.regs_dirty);
 253 out:
 254
 255         return ret;
 256 }
 257 EXPORT_SYMBOL_GPL(load_pdptrs);
 258
 259 static bool pdptrs_changed(struct kvm_vcpu *vcpu)
 260 {
 261         u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 262         bool changed = true;
 263         int r;
 264
 265         if (is_long_mode(vcpu) || !is_pae(vcpu))
 266                 return false;
 267
 268         if (!test_bit(VCPU_EXREG_PDPTR,
 269                       (unsigned long *)&vcpu->arch.regs_avail))
 270                 return true;
 271
 272         r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
 273         if (r < 0)
 274                 goto out;
 275         changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0;
 276 out:
 277
 278         return changed;
 279 }
 280
 281 void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 282 {
 283         if (cr0 & CR0_RESERVED_BITS) {
 284                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 285                        cr0, vcpu->arch.cr0);
 286                 kvm_inject_gp(vcpu, 0);
 287                 return;
 288         }
 289
 290         if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
 291                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 292                 kvm_inject_gp(vcpu, 0);
 293                 return;
 294         }
 295
 296         if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
 297                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 298                        "and a clear PE flag\n");
 299                 kvm_inject_gp(vcpu, 0);
 300                 return;
 301         }
 302
 303         if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
 304 #ifdef CONFIG_X86_64
 305                 if ((vcpu->arch.shadow_efer & EFER_LME)) {
 306                         int cs_db, cs_l;
 307
 308                         if (!is_pae(vcpu)) {
 309                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 310                                        "in long mode while PAE is disabled\n");
 311                                 kvm_inject_gp(vcpu, 0);
 312                                 return;
 313                         }
 314                         kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 315                         if (cs_l) {
 316                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 317                                        "in long mode while CS.L == 1\n");
 318                                 kvm_inject_gp(vcpu, 0);
 319                                 return;
 320
 321                         }
 322                 } else
 323 #endif
 324                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
 325                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 326                                "reserved bits\n");
 327                         kvm_inject_gp(vcpu, 0);
 328                         return;
 329                 }
 330
 331         }
 332
 333         kvm_x86_ops->set_cr0(vcpu, cr0);
 334         vcpu->arch.cr0 = cr0;
 335
 336         kvm_mmu_reset_context(vcpu);
 337         return;
 338 }
 339 EXPORT_SYMBOL_GPL(kvm_set_cr0);
 340
 341 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 342 {
 343         kvm_set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f));
 344         KVMTRACE_1D(LMSW, vcpu,
 345                     (u32)((vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f)),
 346                     handler);
 347 }
 348 EXPORT_SYMBOL_GPL(kvm_lmsw);
 349
 350 void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 351 {
 352         unsigned long old_cr4 = vcpu->arch.cr4;
 353         unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE;
 354
 355         if (cr4 & CR4_RESERVED_BITS) {
 356                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 357                 kvm_inject_gp(vcpu, 0);
 358                 return;
 359         }
 360
 361         if (is_long_mode(vcpu)) {
 362                 if (!(cr4 & X86_CR4_PAE)) {
 363                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 364                                "in long mode\n");
 365                         kvm_inject_gp(vcpu, 0);
 366                         return;
 367                 }
 368         } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
 369                    && ((cr4 ^ old_cr4) & pdptr_bits)
 370                    && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
 371                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 372                 kvm_inject_gp(vcpu, 0);
 373                 return;
 374         }
 375
 376         if (cr4 & X86_CR4_VMXE) {
 377                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 378                 kvm_inject_gp(vcpu, 0);
 379                 return;
 380         }
 381         kvm_x86_ops->set_cr4(vcpu, cr4);
 382         vcpu->arch.cr4 = cr4;
 383         vcpu->arch.mmu.base_role.cr4_pge = (cr4 & X86_CR4_PGE) && !tdp_enabled;
 384         kvm_mmu_reset_context(vcpu);
 385 }
 386 EXPORT_SYMBOL_GPL(kvm_set_cr4);
 387
 388 void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 389 {
 390         if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
 391                 kvm_mmu_sync_roots(vcpu);
 392                 kvm_mmu_flush_tlb(vcpu);
 393                 return;
 394         }
 395
 396         if (is_long_mode(vcpu)) {
 397                 if (cr3 & CR3_L_MODE_RESERVED_BITS) {
 398                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 399                         kvm_inject_gp(vcpu, 0);
 400                         return;
 401                 }
 402         } else {
 403                 if (is_pae(vcpu)) {
 404                         if (cr3 & CR3_PAE_RESERVED_BITS) {
 405                                 printk(KERN_DEBUG
 406                                        "set_cr3: #GP, reserved bits\n");
 407                                 kvm_inject_gp(vcpu, 0);
 408                                 return;
 409                         }
 410                         if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
 411                                 printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 412                                        "reserved bits\n");
 413                                 kvm_inject_gp(vcpu, 0);
 414                                 return;
 415                         }
 416                 }
 417                 /*
 418                  * We don't check reserved bits in nonpae mode, because
 419                  * this isn't enforced, and VMware depends on this.
 420                  */
 421         }
 422
 423         /*
 424          * Does the new cr3 value map to physical memory? (Note, we
 425          * catch an invalid cr3 even in real-mode, because it would
 426          * cause trouble later on when we turn on paging anyway.)
 427          *
 428          * A real CPU would silently accept an invalid cr3 and would
 429          * attempt to use it - with largely undefined (and often hard
 430          * to debug) behavior on the guest side.
 431          */
 432         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 433                 kvm_inject_gp(vcpu, 0);
 434         else {
 435                 vcpu->arch.cr3 = cr3;
 436                 vcpu->arch.mmu.new_cr3(vcpu);
 437         }
 438 }
 439 EXPORT_SYMBOL_GPL(kvm_set_cr3);
 440
 441 void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 442 {
 443         if (cr8 & CR8_RESERVED_BITS) {
 444                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 445                 kvm_inject_gp(vcpu, 0);
 446                 return;
 447         }
 448         if (irqchip_in_kernel(vcpu->kvm))
 449                 kvm_lapic_set_tpr(vcpu, cr8);
 450         else
 451                 vcpu->arch.cr8 = cr8;
 452 }
 453 EXPORT_SYMBOL_GPL(kvm_set_cr8);
 454
 455 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
 456 {
 457         if (irqchip_in_kernel(vcpu->kvm))
 458                 return kvm_lapic_get_cr8(vcpu);
 459         else
 460                 return vcpu->arch.cr8;
 461 }
 462 EXPORT_SYMBOL_GPL(kvm_get_cr8);
 463
 464 static inline u32 bit(int bitno)
 465 {
 466         return 1 << (bitno & 31);
 467 }
 468
 469 /*
 470  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 471  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 472  *
 473  * This list is modified at module load time to reflect the
 474  * capabilities of the host cpu.
 475  */
 476 static u32 msrs_to_save[] = {
 477         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 478         MSR_K6_STAR,
 479 #ifdef CONFIG_X86_64
 480         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
 481 #endif
 482         MSR_IA32_TSC, MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
 483         MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
 484 };
 485
 486 static unsigned num_msrs_to_save;
 487
 488 static u32 emulated_msrs[] = {
 489         MSR_IA32_MISC_ENABLE,
 490 };
 491
 492 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
 493 {
 494         if (efer & efer_reserved_bits) {
 495                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
 496                        efer);
 497                 kvm_inject_gp(vcpu, 0);
 498                 return;
 499         }
 500
 501         if (is_paging(vcpu)
 502             && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) {
 503                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
 504                 kvm_inject_gp(vcpu, 0);
 505                 return;
 506         }
 507
 508         if (efer & EFER_FFXSR) {
 509                 struct kvm_cpuid_entry2 *feat;
 510
 511                 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
 512                 if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT))) {
 513                         printk(KERN_DEBUG "set_efer: #GP, enable FFXSR w/o CPUID capability\n");
 514                         kvm_inject_gp(vcpu, 0);
 515                         return;
 516                 }
 517         }
 518
 519         if (efer & EFER_SVME) {
 520                 struct kvm_cpuid_entry2 *feat;
 521
 522                 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
 523                 if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM))) {
 524                         printk(KERN_DEBUG "set_efer: #GP, enable SVM w/o SVM\n");
 525                         kvm_inject_gp(vcpu, 0);
 526                         return;
 527                 }
 528         }
 529
 530         kvm_x86_ops->set_efer(vcpu, efer);
 531
 532         efer &= ~EFER_LMA;
 533         efer |= vcpu->arch.shadow_efer & EFER_LMA;
 534
 535         vcpu->arch.shadow_efer = efer;
 536
 537         vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
 538         kvm_mmu_reset_context(vcpu);
 539 }
 540
 541 void kvm_enable_efer_bits(u64 mask)
 542 {
 543        efer_reserved_bits &= ~mask;
 544 }
 545 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
 546
 547
 548 /*
 549  * Writes msr value into into the appropriate "register".
 550  * Returns 0 on success, non-0 otherwise.
 551  * Assumes vcpu_load() was already called.
 552  */
 553 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
 554 {
 555         return kvm_x86_ops->set_msr(vcpu, msr_index, data);
 556 }
 557
 558 /*
 559  * Adapt set_msr() to msr_io()'s calling convention
 560  */
 561 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
 562 {
 563         return kvm_set_msr(vcpu, index, *data);
 564 }
 565
 566 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
 567 {
 568         static int version;
 569         struct pvclock_wall_clock wc;
 570         struct timespec now, sys, boot;
 571
 572         if (!wall_clock)
 573                 return;
 574
 575         version++;
 576
 577         kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
 578
 579         /*
 580          * The guest calculates current wall clock time by adding
 581          * system time (updated by kvm_write_guest_time below) to the
 582          * wall clock specified here.  guest system time equals host
 583          * system time for us, thus we must fill in host boot time here.
 584          */
 585         now = current_kernel_time();
 586         ktime_get_ts(&sys);
 587         boot = ns_to_timespec(timespec_to_ns(&now) - timespec_to_ns(&sys));
 588
 589         wc.sec = boot.tv_sec;
 590         wc.nsec = boot.tv_nsec;
 591         wc.version = version;
 592
 593         kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
 594
 595         version++;
 596         kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
 597 }
 598
 599 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
 600 {
 601         uint32_t quotient, remainder;
 602
 603         /* Don't try to replace with do_div(), this one calculates
 604          * "(dividend << 32) / divisor" */
 605         __asm__ ( "divl %4"
 606                   : "=a" (quotient), "=d" (remainder)
 607                   : "0" (0), "1" (dividend), "r" (divisor) );
 608         return quotient;
 609 }
 610
 611 static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info *hv_clock)
 612 {
 613         uint64_t nsecs = 1000000000LL;
 614         int32_t  shift = 0;
 615         uint64_t tps64;
 616         uint32_t tps32;
 617
 618         tps64 = tsc_khz * 1000LL;
 619         while (tps64 > nsecs*2) {
 620                 tps64 >>= 1;
 621                 shift--;
 622         }
 623
 624         tps32 = (uint32_t)tps64;
 625         while (tps32 <= (uint32_t)nsecs) {
 626                 tps32 <<= 1;
 627                 shift++;
 628         }
 629
 630         hv_clock->tsc_shift = shift;
 631         hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32);
 632
 633         pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n",
 634                  __func__, tsc_khz, hv_clock->tsc_shift,
 635                  hv_clock->tsc_to_system_mul);
 636 }
 637
 638 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
 639
 640 static void kvm_write_guest_time(struct kvm_vcpu *v)
 641 {
 642         struct timespec ts;
 643         unsigned long flags;
 644         struct kvm_vcpu_arch *vcpu = &v->arch;
 645         void *shared_kaddr;
 646         unsigned long this_tsc_khz;
 647
 648         if ((!vcpu->time_page))
 649                 return;
 650
 651         this_tsc_khz = get_cpu_var(cpu_tsc_khz);
 652         if (unlikely(vcpu->hv_clock_tsc_khz != this_tsc_khz)) {
 653                 kvm_set_time_scale(this_tsc_khz, &vcpu->hv_clock);
 654                 vcpu->hv_clock_tsc_khz = this_tsc_khz;
 655         }
 656         put_cpu_var(cpu_tsc_khz);
 657
 658         /* Keep irq disabled to prevent changes to the clock */
 659         local_irq_save(flags);
 660         kvm_get_msr(v, MSR_IA32_TSC, &vcpu->hv_clock.tsc_timestamp);
 661         ktime_get_ts(&ts);
 662         local_irq_restore(flags);
 663
 664         /* With all the info we got, fill in the values */
 665
 666         vcpu->hv_clock.system_time = ts.tv_nsec +
 667                                      (NSEC_PER_SEC * (u64)ts.tv_sec);
 668         /*
 669          * The interface expects us to write an even number signaling that the
 670          * update is finished. Since the guest won't see the intermediate
 671          * state, we just increase by 2 at the end.
 672          */
 673         vcpu->hv_clock.version += 2;
 674
 675         shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);
 676
 677         memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
 678                sizeof(vcpu->hv_clock));
 679
 680         kunmap_atomic(shared_kaddr, KM_USER0);
 681
 682         mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
 683 }
 684
 685 static int kvm_request_guest_time_update(struct kvm_vcpu *v)
 686 {
 687         struct kvm_vcpu_arch *vcpu = &v->arch;
 688
 689         if (!vcpu->time_page)
 690                 return 0;
 691         set_bit(KVM_REQ_KVMCLOCK_UPDATE, &v->requests);
 692         return 1;
 693 }
 694
 695 static bool msr_mtrr_valid(unsigned msr)
 696 {
 697         switch (msr) {
 698         case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
 699         case MSR_MTRRfix64K_00000:
 700         case MSR_MTRRfix16K_80000:
 701         case MSR_MTRRfix16K_A0000:
 702         case MSR_MTRRfix4K_C0000:
 703         case MSR_MTRRfix4K_C8000:
 704         case MSR_MTRRfix4K_D0000:
 705         case MSR_MTRRfix4K_D8000:
 706         case MSR_MTRRfix4K_E0000:
 707         case MSR_MTRRfix4K_E8000:
 708         case MSR_MTRRfix4K_F0000:
 709         case MSR_MTRRfix4K_F8000:
 710         case MSR_MTRRdefType:
 711         case MSR_IA32_CR_PAT:
 712                 return true;
 713         case 0x2f8:
 714                 return true;
 715         }
 716         return false;
 717 }
 718
 719 static bool valid_pat_type(unsigned t)
 720 {
 721         return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
 722 }
 723
 724 static bool valid_mtrr_type(unsigned t)
 725 {
 726         return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
 727 }
 728
 729 static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 730 {
 731         int i;
 732
 733         if (!msr_mtrr_valid(msr))
 734                 return false;
 735
 736         if (msr == MSR_IA32_CR_PAT) {
 737                 for (i = 0; i < 8; i++)
 738                         if (!valid_pat_type((data >> (i * 8)) & 0xff))
 739                                 return false;
 740                 return true;
 741         } else if (msr == MSR_MTRRdefType) {
 742                 if (data & ~0xcff)
 743                         return false;
 744                 return valid_mtrr_type(data & 0xff);
 745         } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
 746                 for (i = 0; i < 8 ; i++)
 747                         if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
 748                                 return false;
 749                 return true;
 750         }
 751
 752         /* variable MTRRs */
 753         return valid_mtrr_type(data & 0xff);
 754 }
 755
 756 static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 757 {
 758         u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
 759
 760         if (!mtrr_valid(vcpu, msr, data))
 761                 return 1;
 762
 763         if (msr == MSR_MTRRdefType) {
 764                 vcpu->arch.mtrr_state.def_type = data;
 765                 vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
 766         } else if (msr == MSR_MTRRfix64K_00000)
 767                 p[0] = data;
 768         else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
 769                 p[1 + msr - MSR_MTRRfix16K_80000] = data;
 770         else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
 771                 p[3 + msr - MSR_MTRRfix4K_C0000] = data;
 772         else if (msr == MSR_IA32_CR_PAT)
 773                 vcpu->arch.pat = data;
 774         else {  /* Variable MTRRs */
 775                 int idx, is_mtrr_mask;
 776                 u64 *pt;
 777
 778                 idx = (msr - 0x200) / 2;
 779                 is_mtrr_mask = msr - 0x200 - 2 * idx;
 780                 if (!is_mtrr_mask)
 781                         pt =
 782                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
 783                 else
 784                         pt =
 785                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
 786                 *pt = data;
 787         }
 788
 789         kvm_mmu_reset_context(vcpu);
 790         return 0;
 791 }
 792
 793 static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 794 {
 795         u64 mcg_cap = vcpu->arch.mcg_cap;
 796         unsigned bank_num = mcg_cap & 0xff;
 797
 798         switch (msr) {
 799         case MSR_IA32_MCG_STATUS:
 800                 vcpu->arch.mcg_status = data;
 801                 break;
 802         case MSR_IA32_MCG_CTL:
 803                 if (!(mcg_cap & MCG_CTL_P))
 804                         return 1;
 805                 if (data != 0 && data != ~(u64)0)
 806                         return -1;
 807                 vcpu->arch.mcg_ctl = data;
 808                 break;
 809         default:
 810                 if (msr >= MSR_IA32_MC0_CTL &&
 811                     msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
 812                         u32 offset = msr - MSR_IA32_MC0_CTL;
 813                         /* only 0 or all 1s can be written to IA32_MCi_CTL */
 814                         if ((offset & 0x3) == 0 &&
 815                             data != 0 && data != ~(u64)0)
 816                                 return -1;
 817                         vcpu->arch.mce_banks[offset] = data;
 818                         break;
 819                 }
 820                 return 1;
 821         }
 822         return 0;
 823 }
 824
 825 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 826 {
 827         switch (msr) {
 828         case MSR_EFER:
 829                 set_efer(vcpu, data);
 830                 break;
 831         case MSR_IA32_DEBUGCTLMSR:
 832                 if (!data) {
 833                         /* We support the non-activated case already */
 834                         break;
 835                 } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
 836                         /* Values other than LBR and BTF are vendor-specific,
 837                            thus reserved and should throw a #GP */
 838                         return 1;
 839                 }
 840                 pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
 841                         __func__, data);
 842                 break;
 843         case MSR_IA32_UCODE_REV:
 844         case MSR_IA32_UCODE_WRITE:
 845         case MSR_VM_HSAVE_PA:
 846                 break;
 847         case 0x200 ... 0x2ff:
 848                 return set_msr_mtrr(vcpu, msr, data);
 849         case MSR_IA32_APICBASE:
 850                 kvm_set_apic_base(vcpu, data);
 851                 break;
 852         case MSR_IA32_MISC_ENABLE:
 853                 vcpu->arch.ia32_misc_enable_msr = data;
 854                 break;
 855         case MSR_KVM_WALL_CLOCK:
 856                 vcpu->kvm->arch.wall_clock = data;
 857                 kvm_write_wall_clock(vcpu->kvm, data);
 858                 break;
 859         case MSR_KVM_SYSTEM_TIME: {
 860                 if (vcpu->arch.time_page) {
 861                         kvm_release_page_dirty(vcpu->arch.time_page);
 862                         vcpu->arch.time_page = NULL;
 863                 }
 864
 865                 vcpu->arch.time = data;
 866
 867                 /* we verify if the enable bit is set... */
 868                 if (!(data & 1))
 869                         break;
 870
 871                 /* ...but clean it before doing the actual write */
 872                 vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
 873
 874                 vcpu->arch.time_page =
 875                                 gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
 876
 877                 if (is_error_page(vcpu->arch.time_page)) {
 878                         kvm_release_page_clean(vcpu->arch.time_page);
 879                         vcpu->arch.time_page = NULL;
 880                 }
 881
 882                 kvm_request_guest_time_update(vcpu);
 883                 break;
 884         }
 885         case MSR_IA32_MCG_CTL:
 886         case MSR_IA32_MCG_STATUS:
 887         case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
 888                 return set_msr_mce(vcpu, msr, data);
 889         default:
 890                 pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", msr, data);
 891                 return 1;
 892         }
 893         return 0;
 894 }
 895 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
 896
 897
 898 /*
 899  * Reads an msr value (of 'msr_index') into 'pdata'.
 900  * Returns 0 on success, non-0 otherwise.
 901  * Assumes vcpu_load() was already called.
 902  */
 903 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 904 {
 905         return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
 906 }
 907
 908 static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 909 {
 910         u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
 911
 912         if (!msr_mtrr_valid(msr))
 913                 return 1;
 914
 915         if (msr == MSR_MTRRdefType)
 916                 *pdata = vcpu->arch.mtrr_state.def_type +
 917                          (vcpu->arch.mtrr_state.enabled << 10);
 918         else if (msr == MSR_MTRRfix64K_00000)
 919                 *pdata = p[0];
 920         else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
 921                 *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
 922         else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
 923                 *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
 924         else if (msr == MSR_IA32_CR_PAT)
 925                 *pdata = vcpu->arch.pat;
 926         else {  /* Variable MTRRs */
 927                 int idx, is_mtrr_mask;
 928                 u64 *pt;
 929
 930                 idx = (msr - 0x200) / 2;
 931                 is_mtrr_mask = msr - 0x200 - 2 * idx;
 932                 if (!is_mtrr_mask)
 933                         pt =
 934                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
 935                 else
 936                         pt =
 937                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
 938                 *pdata = *pt;
 939         }
 940
 941         return 0;
 942 }
 943
 944 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 945 {
 946         u64 data;
 947         u64 mcg_cap = vcpu->arch.mcg_cap;
 948         unsigned bank_num = mcg_cap & 0xff;
 949
 950         switch (msr) {
 951         case MSR_IA32_P5_MC_ADDR:
 952         case MSR_IA32_P5_MC_TYPE:
 953                 data = 0;
 954                 break;
 955         case MSR_IA32_MCG_CAP:
 956                 data = vcpu->arch.mcg_cap;
 957                 break;
 958         case MSR_IA32_MCG_CTL:
 959                 if (!(mcg_cap & MCG_CTL_P))
 960                         return 1;
 961                 data = vcpu->arch.mcg_ctl;
 962                 break;
 963         case MSR_IA32_MCG_STATUS:
 964                 data = vcpu->arch.mcg_status;
 965                 break;
 966         default:
 967                 if (msr >= MSR_IA32_MC0_CTL &&
 968                     msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
 969                         u32 offset = msr - MSR_IA32_MC0_CTL;
 970                         data = vcpu->arch.mce_banks[offset];
 971                         break;
 972                 }
 973                 return 1;
 974         }
 975         *pdata = data;
 976         return 0;
 977 }
 978
 979 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 980 {
 981         u64 data;
 982
 983         switch (msr) {
 984         case MSR_IA32_PLATFORM_ID:
 985         case MSR_IA32_UCODE_REV:
 986         case MSR_IA32_EBL_CR_POWERON:
 987         case MSR_IA32_DEBUGCTLMSR:
 988         case MSR_IA32_LASTBRANCHFROMIP:
 989         case MSR_IA32_LASTBRANCHTOIP:
 990         case MSR_IA32_LASTINTFROMIP:
 991         case MSR_IA32_LASTINTTOIP:
 992         case MSR_K8_SYSCFG:
 993         case MSR_K7_HWCR:
 994         case MSR_VM_HSAVE_PA:
 995         case MSR_P6_EVNTSEL0:
 996         case MSR_P6_EVNTSEL1:
 997         case MSR_K7_EVNTSEL0:
 998                 data = 0;
 999                 break;
1000         case MSR_MTRRcap:
1001                 data = 0x500 | KVM_NR_VAR_MTRR;
1002                 break;
1003         case 0x200 ... 0x2ff:
1004                 return get_msr_mtrr(vcpu, msr, pdata);
1005         case 0xcd: /* fsb frequency */
1006                 data = 3;
1007                 break;
1008         case MSR_IA32_APICBASE:
1009                 data = kvm_get_apic_base(vcpu);
1010                 break;
1011         case MSR_IA32_MISC_ENABLE:
1012                 data = vcpu->arch.ia32_misc_enable_msr;
1013                 break;
1014         case MSR_IA32_PERF_STATUS:
1015                 /* TSC increment by tick */
1016                 data = 1000ULL;
1017                 /* CPU multiplier */
1018                 data |= (((uint64_t)4ULL) << 40);
1019                 break;
1020         case MSR_EFER:
1021                 data = vcpu->arch.shadow_efer;
1022                 break;
1023         case MSR_KVM_WALL_CLOCK:
1024                 data = vcpu->kvm->arch.wall_clock;
1025                 break;
1026         case MSR_KVM_SYSTEM_TIME:
1027                 data = vcpu->arch.time;
1028                 break;
1029         case MSR_IA32_P5_MC_ADDR:
1030         case MSR_IA32_P5_MC_TYPE:
1031         case MSR_IA32_MCG_CAP:
1032         case MSR_IA32_MCG_CTL:
1033         case MSR_IA32_MCG_STATUS:
1034         case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
1035                 return get_msr_mce(vcpu, msr, pdata);
1036         default:
1037                 pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
1038                 return 1;
1039         }
1040         *pdata = data;
1041         return 0;
1042 }
1043 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1044
1045 /*
1046  * Read or write a bunch of msrs. All parameters are kernel addresses.
1047  *
1048  * @return number of msrs set successfully.
1049  */
1050 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
1051                     struct kvm_msr_entry *entries,
1052                     int (*do_msr)(struct kvm_vcpu *vcpu,
1053                                   unsigned index, u64 *data))
1054 {
1055         int i;
1056
1057         vcpu_load(vcpu);
1058
1059         down_read(&vcpu->kvm->slots_lock);
1060         for (i = 0; i < msrs->nmsrs; ++i)
1061                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1062                         break;
1063         up_read(&vcpu->kvm->slots_lock);
1064
1065         vcpu_put(vcpu);
1066
1067         return i;
1068 }
1069
1070 /*
1071  * Read or write a bunch of msrs. Parameters are user addresses.
1072  *
1073  * @return number of msrs set successfully.
1074  */
1075 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
1076                   int (*do_msr)(struct kvm_vcpu *vcpu,
1077                                 unsigned index, u64 *data),
1078                   int writeback)
1079 {
1080         struct kvm_msrs msrs;
1081         struct kvm_msr_entry *entries;
1082         int r, n;
1083         unsigned size;
1084
1085         r = -EFAULT;
1086         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1087                 goto out;
1088
1089         r = -E2BIG;
1090         if (msrs.nmsrs >= MAX_IO_MSRS)
1091                 goto out;
1092
1093         r = -ENOMEM;
1094         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1095         entries = vmalloc(size);
1096         if (!entries)
1097                 goto out;
1098
1099         r = -EFAULT;
1100         if (copy_from_user(entries, user_msrs->entries, size))
1101                 goto out_free;
1102
1103         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
1104         if (r < 0)
1105                 goto out_free;
1106
1107         r = -EFAULT;
1108         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1109                 goto out_free;
1110
1111         r = n;
1112
1113 out_free:
1114         vfree(entries);
1115 out:
1116         return r;
1117 }
1118
1119 int kvm_dev_ioctl_check_extension(long ext)
1120 {
1121         int r;
1122
1123         switch (ext) {
1124         case KVM_CAP_IRQCHIP:
1125         case KVM_CAP_HLT:
1126         case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
1127         case KVM_CAP_SET_TSS_ADDR:
1128         case KVM_CAP_EXT_CPUID:
1129         case KVM_CAP_CLOCKSOURCE:
1130         case KVM_CAP_PIT:
1131         case KVM_CAP_NOP_IO_DELAY:
1132         case KVM_CAP_MP_STATE:
1133         case KVM_CAP_SYNC_MMU:
1134         case KVM_CAP_REINJECT_CONTROL:
1135         case KVM_CAP_IRQ_INJECT_STATUS:
1136         case KVM_CAP_ASSIGN_DEV_IRQ:
1137         case KVM_CAP_IRQFD:
1138         case KVM_CAP_PIT2:
1139                 r = 1;
1140                 break;
1141         case KVM_CAP_COALESCED_MMIO:
1142                 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
1143                 break;
1144         case KVM_CAP_VAPIC:
1145                 r = !kvm_x86_ops->cpu_has_accelerated_tpr();
1146                 break;
1147         case KVM_CAP_NR_VCPUS:
1148                 r = KVM_MAX_VCPUS;
1149                 break;
1150         case KVM_CAP_NR_MEMSLOTS:
1151                 r = KVM_MEMORY_SLOTS;
1152                 break;
1153         case KVM_CAP_PV_MMU:
1154                 r = !tdp_enabled;
1155                 break;
1156         case KVM_CAP_IOMMU:
1157                 r = iommu_found();
1158                 break;
1159         case KVM_CAP_MCE:
1160                 r = KVM_MAX_MCE_BANKS;
1161                 break;
1162         default:
1163                 r = 0;
1164                 break;
1165         }
1166         return r;
1167
1168 }
1169
1170 long kvm_arch_dev_ioctl(struct file *filp,
1171                         unsigned int ioctl, unsigned long arg)
1172 {
1173         void __user *argp = (void __user *)arg;
1174         long r;
1175
1176         switch (ioctl) {
1177         case KVM_GET_MSR_INDEX_LIST: {
1178                 struct kvm_msr_list __user *user_msr_list = argp;
1179                 struct kvm_msr_list msr_list;
1180                 unsigned n;
1181
1182                 r = -EFAULT;
1183                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
1184                         goto out;
1185                 n = msr_list.nmsrs;
1186                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
1187                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
1188                         goto out;
1189                 r = -E2BIG;
1190                 if (n < msr_list.nmsrs)
1191                         goto out;
1192                 r = -EFAULT;
1193                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
1194                                  num_msrs_to_save * sizeof(u32)))
1195                         goto out;
1196                 if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
1197                                  &emulated_msrs,
1198                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
1199                         goto out;
1200                 r = 0;
1201                 break;
1202         }
1203         case KVM_GET_SUPPORTED_CPUID: {
1204                 struct kvm_cpuid2 __user *cpuid_arg = argp;
1205                 struct kvm_cpuid2 cpuid;
1206
1207                 r = -EFAULT;
1208                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1209                         goto out;
1210                 r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
1211                                                       cpuid_arg->entries);
1212                 if (r)
1213                         goto out;
1214
1215                 r = -EFAULT;
1216                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
1217                         goto out;
1218                 r = 0;
1219                 break;
1220         }
1221         case KVM_X86_GET_MCE_CAP_SUPPORTED: {
1222                 u64 mce_cap;
1223
1224                 mce_cap = KVM_MCE_CAP_SUPPORTED;
1225                 r = -EFAULT;
1226                 if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
1227                         goto out;
1228                 r = 0;
1229                 break;
1230         }
1231         default:
1232                 r = -EINVAL;
1233         }
1234 out:
1235         return r;
1236 }
1237
1238 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1239 {
1240         kvm_x86_ops->vcpu_load(vcpu, cpu);
1241         kvm_request_guest_time_update(vcpu);
1242 }
1243
1244 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1245 {
1246         kvm_x86_ops->vcpu_put(vcpu);
1247         kvm_put_guest_fpu(vcpu);
1248 }
1249
1250 static int is_efer_nx(void)
1251 {
1252         unsigned long long efer = 0;
1253
1254         rdmsrl_safe(MSR_EFER, &efer);
1255         return efer & EFER_NX;
1256 }
1257
1258 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
1259 {
1260         int i;
1261         struct kvm_cpuid_entry2 *e, *entry;
1262
1263         entry = NULL;
1264         for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
1265                 e = &vcpu->arch.cpuid_entries[i];
1266                 if (e->function == 0x80000001) {
1267                         entry = e;
1268                         break;
1269                 }
1270         }
1271         if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
1272                 entry->edx &= ~(1 << 20);
1273                 printk(KERN_INFO "kvm: guest NX capability removed\n");
1274         }
1275 }
1276
1277 /* when an old userspace process fills a new kernel module */
1278 static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
1279                                     struct kvm_cpuid *cpuid,
1280                                     struct kvm_cpuid_entry __user *entries)
1281 {
1282         int r, i;
1283         struct kvm_cpuid_entry *cpuid_entries;
1284
1285         r = -E2BIG;
1286         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
1287                 goto out;
1288         r = -ENOMEM;
1289         cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
1290         if (!cpuid_entries)
1291                 goto out;
1292         r = -EFAULT;
1293         if (copy_from_user(cpuid_entries, entries,
1294                            cpuid->nent * sizeof(struct kvm_cpuid_entry)))
1295                 goto out_free;
1296         for (i = 0; i < cpuid->nent; i++) {
1297                 vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
1298                 vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
1299                 vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
1300                 vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
1301                 vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
1302                 vcpu->arch.cpuid_entries[i].index = 0;
1303                 vcpu->arch.cpuid_entries[i].flags = 0;
1304                 vcpu->arch.cpuid_entries[i].padding[0] = 0;
1305                 vcpu->arch.cpuid_entries[i].padding[1] = 0;
1306                 vcpu->arch.cpuid_entries[i].padding[2] = 0;
1307         }
1308         vcpu->arch.cpuid_nent = cpuid->nent;
1309         cpuid_fix_nx_cap(vcpu);
1310         r = 0;
1311
1312 out_free:
1313         vfree(cpuid_entries);
1314 out:
1315         return r;
1316 }
1317
1318 static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
1319                                      struct kvm_cpuid2 *cpuid,
1320                                      struct kvm_cpuid_entry2 __user *entries)
1321 {
1322         int r;
1323
1324         r = -E2BIG;
1325         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
1326                 goto out;
1327         r = -EFAULT;
1328         if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
1329                            cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
1330                 goto out;
1331         vcpu->arch.cpuid_nent = cpuid->nent;
1332         return 0;
1333
1334 out:
1335         return r;
1336 }
1337
1338 static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
1339                                      struct kvm_cpuid2 *cpuid,
1340                                      struct kvm_cpuid_entry2 __user *entries)
1341 {
1342         int r;
1343
1344         r = -E2BIG;
1345         if (cpuid->nent < vcpu->arch.cpuid_nent)
1346                 goto out;
1347         r = -EFAULT;
1348         if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
1349                          vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
1350                 goto out;
1351         return 0;
1352
1353 out:
1354         cpuid->nent = vcpu->arch.cpuid_nent;
1355         return r;
1356 }
1357
1358 static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
1359                            u32 index)
1360 {
1361         entry->function = function;
1362         entry->index = index;
1363         cpuid_count(entry->function, entry->index,
1364                     &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
1365         entry->flags = 0;
1366 }
1367
1368 #define F(x) bit(X86_FEATURE_##x)
1369
1370 static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
1371                          u32 index, int *nent, int maxnent)
1372 {
1373         unsigned f_nx = is_efer_nx() ? F(NX) : 0;
1374 #ifdef CONFIG_X86_64
1375         unsigned f_lm = F(LM);
1376 #else
1377         unsigned f_lm = 0;
1378 #endif
1379
1380         /* cpuid 1.edx */
1381         const u32 kvm_supported_word0_x86_features =
1382                 F(FPU) | F(VME) | F(DE) | F(PSE) |
1383                 F(TSC) | F(MSR) | F(PAE) | F(MCE) |
1384                 F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
1385                 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
1386                 F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
1387                 0 /* Reserved, DS, ACPI */ | F(MMX) |
1388                 F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
1389                 0 /* HTT, TM, Reserved, PBE */;
1390         /* cpuid 0x80000001.edx */
1391         const u32 kvm_supported_word1_x86_features =
1392                 F(FPU) | F(VME) | F(DE) | F(PSE) |
1393                 F(TSC) | F(MSR) | F(PAE) | F(MCE) |
1394                 F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
1395                 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
1396                 F(PAT) | F(PSE36) | 0 /* Reserved */ |
1397                 f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
1398                 F(FXSR) | F(FXSR_OPT) | 0 /* GBPAGES */ | 0 /* RDTSCP */ |
1399                 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
1400         /* cpuid 1.ecx */
1401         const u32 kvm_supported_word4_x86_features =
1402                 F(XMM3) | 0 /* Reserved, DTES64, MONITOR */ |
1403                 0 /* DS-CPL, VMX, SMX, EST */ |
1404                 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
1405                 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
1406                 0 /* Reserved, DCA */ | F(XMM4_1) |
1407                 F(XMM4_2) | 0 /* x2APIC */ | F(MOVBE) | F(POPCNT) |
1408                 0 /* Reserved, XSAVE, OSXSAVE */;
1409         /* cpuid 0x80000001.ecx */
1410         const u32 kvm_supported_word6_x86_features =
1411                 F(LAHF_LM) | F(CMP_LEGACY) | F(SVM) | 0 /* ExtApicSpace */ |
1412                 F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
1413                 F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(SSE5) |
1414                 0 /* SKINIT */ | 0 /* WDT */;
1415
1416         /* all calls to cpuid_count() should be made on the same cpu */
1417         get_cpu();
1418         do_cpuid_1_ent(entry, function, index);
1419         ++*nent;
1420
1421         switch (function) {
1422         case 0:
1423                 entry->eax = min(entry->eax, (u32)0xb);
1424                 break;
1425         case 1:
1426                 entry->edx &= kvm_supported_word0_x86_features;
1427                 entry->ecx &= kvm_supported_word4_x86_features;
1428                 break;
1429         /* function 2 entries are STATEFUL. That is, repeated cpuid commands
1430          * may return different values. This forces us to get_cpu() before
1431          * issuing the first command, and also to emulate this annoying behavior
1432          * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
1433         case 2: {
1434                 int t, times = entry->eax & 0xff;
1435
1436                 entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
1437                 entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
1438                 for (t = 1; t < times && *nent < maxnent; ++t) {
1439                         do_cpuid_1_ent(&entry[t], function, 0);
1440                         entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
1441                         ++*nent;
1442                 }
1443                 break;
1444         }
1445         /* function 4 and 0xb have additional index. */
1446         case 4: {
1447                 int i, cache_type;
1448
1449                 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1450                 /* read more entries until cache_type is zero */
1451                 for (i = 1; *nent < maxnent; ++i) {
1452                         cache_type = entry[i - 1].eax & 0x1f;
1453                         if (!cache_type)
1454                                 break;
1455                         do_cpuid_1_ent(&entry[i], function, i);
1456                         entry[i].flags |=
1457                                KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1458                         ++*nent;
1459                 }
1460                 break;
1461         }
1462         case 0xb: {
1463                 int i, level_type;
1464
1465                 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1466                 /* read more entries until level_type is zero */
1467                 for (i = 1; *nent < maxnent; ++i) {
1468                         level_type = entry[i - 1].ecx & 0xff00;
1469                         if (!level_type)
1470                                 break;
1471                         do_cpuid_1_ent(&entry[i], function, i);
1472                         entry[i].flags |=
1473                                KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1474                         ++*nent;
1475                 }
1476                 break;
1477         }
1478         case 0x80000000:
1479                 entry->eax = min(entry->eax, 0x8000001a);
1480                 break;
1481         case 0x80000001:
1482                 entry->edx &= kvm_supported_word1_x86_features;
1483                 entry->ecx &= kvm_supported_word6_x86_features;
1484                 break;
1485         }
1486         put_cpu();
1487 }
1488
1489 #undef F
1490
1491 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
1492                                      struct kvm_cpuid_entry2 __user *entries)
1493 {
1494         struct kvm_cpuid_entry2 *cpuid_entries;
1495         int limit, nent = 0, r = -E2BIG;
1496         u32 func;
1497
1498         if (cpuid->nent < 1)
1499                 goto out;
1500         r = -ENOMEM;
1501         cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
1502         if (!cpuid_entries)
1503                 goto out;
1504
1505         do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
1506         limit = cpuid_entries[0].eax;
1507         for (func = 1; func <= limit && nent < cpuid->nent; ++func)
1508                 do_cpuid_ent(&cpuid_entries[nent], func, 0,
1509                              &nent, cpuid->nent);
1510         r = -E2BIG;
1511         if (nent >= cpuid->nent)
1512                 goto out_free;
1513
1514         do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
1515         limit = cpuid_entries[nent - 1].eax;
1516         for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
1517                 do_cpuid_ent(&cpuid_entries[nent], func, 0,
1518                              &nent, cpuid->nent);
1519         r = -E2BIG;
1520         if (nent >= cpuid->nent)
1521                 goto out_free;
1522
1523         r = -EFAULT;
1524         if (copy_to_user(entries, cpuid_entries,
1525                          nent * sizeof(struct kvm_cpuid_entry2)))
1526                 goto out_free;
1527         cpuid->nent = nent;
1528         r = 0;
1529
1530 out_free:
1531         vfree(cpuid_entries);
1532 out:
1533         return r;
1534 }
1535
1536 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
1537                                     struct kvm_lapic_state *s)
1538 {
1539         vcpu_load(vcpu);
1540         memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
1541         vcpu_put(vcpu);
1542
1543         return 0;
1544 }
1545
1546 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
1547                                     struct kvm_lapic_state *s)
1548 {
1549         vcpu_load(vcpu);
1550         memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
1551         kvm_apic_post_state_restore(vcpu);
1552         vcpu_put(vcpu);
1553
1554         return 0;
1555 }
1556
1557 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
1558                                     struct kvm_interrupt *irq)
1559 {
1560         if (irq->irq < 0 || irq->irq >= 256)
1561                 return -EINVAL;
1562         if (irqchip_in_kernel(vcpu->kvm))
1563                 return -ENXIO;
1564         vcpu_load(vcpu);
1565
1566         kvm_queue_interrupt(vcpu, irq->irq, false);
1567
1568         vcpu_put(vcpu);
1569
1570         return 0;
1571 }
1572
1573 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
1574 {
1575         vcpu_load(vcpu);
1576         kvm_inject_nmi(vcpu);
1577         vcpu_put(vcpu);
1578
1579         return 0;
1580 }
1581
1582 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
1583                                            struct kvm_tpr_access_ctl *tac)
1584 {
1585         if (tac->flags)
1586                 return -EINVAL;
1587         vcpu->arch.tpr_access_reporting = !!tac->enabled;
1588         return 0;
1589 }
1590
1591 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
1592                                         u64 mcg_cap)
1593 {
1594         int r;
1595         unsigned bank_num = mcg_cap & 0xff, bank;
1596
1597         r = -EINVAL;
1598         if (!bank_num)
1599                 goto out;
1600         if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
1601                 goto out;
1602         r = 0;
1603         vcpu->arch.mcg_cap = mcg_cap;
1604         /* Init IA32_MCG_CTL to all 1s */
1605         if (mcg_cap & MCG_CTL_P)
1606                 vcpu->arch.mcg_ctl = ~(u64)0;
1607         /* Init IA32_MCi_CTL to all 1s */
1608         for (bank = 0; bank < bank_num; bank++)
1609                 vcpu->arch.mce_banks[bank*4] = ~(u64)0;
1610 out:
1611         return r;
1612 }
1613
1614 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
1615                                       struct kvm_x86_mce *mce)
1616 {
1617         u64 mcg_cap = vcpu->arch.mcg_cap;
1618         unsigned bank_num = mcg_cap & 0xff;
1619         u64 *banks = vcpu->arch.mce_banks;
1620
1621         if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
1622                 return -EINVAL;
1623         /*
1624          * if IA32_MCG_CTL is not all 1s, the uncorrected error
1625          * reporting is disabled
1626          */
1627         if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
1628             vcpu->arch.mcg_ctl != ~(u64)0)
1629                 return 0;
1630         banks += 4 * mce->bank;
1631         /*
1632          * if IA32_MCi_CTL is not all 1s, the uncorrected error
1633          * reporting is disabled for the bank
1634          */
1635         if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
1636                 return 0;
1637         if (mce->status & MCI_STATUS_UC) {
1638                 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
1639                     !(vcpu->arch.cr4 & X86_CR4_MCE)) {
1640                         printk(KERN_DEBUG "kvm: set_mce: "
1641                                "injects mce exception while "
1642                                "previous one is in progress!\n");
1643                         set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
1644                         return 0;
1645                 }
1646                 if (banks[1] & MCI_STATUS_VAL)
1647                         mce->status |= MCI_STATUS_OVER;
1648                 banks[2] = mce->addr;
1649                 banks[3] = mce->misc;
1650                 vcpu->arch.mcg_status = mce->mcg_status;
1651                 banks[1] = mce->status;
1652                 kvm_queue_exception(vcpu, MC_VECTOR);
1653         } else if (!(banks[1] & MCI_STATUS_VAL)
1654                    || !(banks[1] & MCI_STATUS_UC)) {
1655                 if (banks[1] & MCI_STATUS_VAL)
1656                         mce->status |= MCI_STATUS_OVER;
1657                 banks[2] = mce->addr;
1658                 banks[3] = mce->misc;
1659                 banks[1] = mce->status;
1660         } else
1661                 banks[1] |= MCI_STATUS_OVER;
1662         return 0;
1663 }
1664
1665 long kvm_arch_vcpu_ioctl(struct file *filp,
1666                          unsigned int ioctl, unsigned long arg)
1667 {
1668         struct kvm_vcpu *vcpu = filp->private_data;
1669         void __user *argp = (void __user *)arg;
1670         int r;
1671         struct kvm_lapic_state *lapic = NULL;
1672
1673         switch (ioctl) {
1674         case KVM_GET_LAPIC: {
1675                 lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
1676
1677                 r = -ENOMEM;
1678                 if (!lapic)
1679                         goto out;
1680                 r = kvm_vcpu_ioctl_get_lapic(vcpu, lapic);
1681                 if (r)
1682                         goto out;
1683                 r = -EFAULT;
1684                 if (copy_to_user(argp, lapic, sizeof(struct kvm_lapic_state)))
1685                         goto out;
1686                 r = 0;
1687                 break;
1688         }
1689         case KVM_SET_LAPIC: {
1690                 lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
1691                 r = -ENOMEM;
1692                 if (!lapic)
1693                         goto out;
1694                 r = -EFAULT;
1695                 if (copy_from_user(lapic, argp, sizeof(struct kvm_lapic_state)))
1696                         goto out;
1697                 r = kvm_vcpu_ioctl_set_lapic(vcpu, lapic);
1698                 if (r)
1699                         goto out;
1700                 r = 0;
1701                 break;
1702         }
1703         case KVM_INTERRUPT: {
1704                 struct kvm_interrupt irq;
1705
1706                 r = -EFAULT;
1707                 if (copy_from_user(&irq, argp, sizeof irq))
1708                         goto out;
1709                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1710                 if (r)
1711                         goto out;
1712                 r = 0;
1713                 break;
1714         }
1715         case KVM_NMI: {
1716                 r = kvm_vcpu_ioctl_nmi(vcpu);
1717                 if (r)
1718                         goto out;
1719                 r = 0;
1720                 break;
1721         }
1722         case KVM_SET_CPUID: {
1723                 struct kvm_cpuid __user *cpuid_arg = argp;
1724                 struct kvm_cpuid cpuid;
1725
1726                 r = -EFAULT;
1727                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1728                         goto out;
1729                 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
1730                 if (r)
1731                         goto out;
1732                 break;
1733         }
1734         case KVM_SET_CPUID2: {
1735                 struct kvm_cpuid2 __user *cpuid_arg = argp;
1736                 struct kvm_cpuid2 cpuid;
1737
1738                 r = -EFAULT;
1739                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1740                         goto out;
1741                 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
1742                                               cpuid_arg->entries);
1743                 if (r)
1744                         goto out;
1745                 break;
1746         }
1747         case KVM_GET_CPUID2: {
1748                 struct kvm_cpuid2 __user *cpuid_arg = argp;
1749                 struct kvm_cpuid2 cpuid;
1750
1751                 r = -EFAULT;
1752                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1753                         goto out;
1754                 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
1755                                               cpuid_arg->entries);
1756                 if (r)
1757                         goto out;
1758                 r = -EFAULT;
1759                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
1760                         goto out;
1761                 r = 0;
1762                 break;
1763         }
1764         case KVM_GET_MSRS:
1765                 r = msr_io(vcpu, argp, kvm_get_msr, 1);
1766                 break;
1767         case KVM_SET_MSRS:
1768                 r = msr_io(vcpu, argp, do_set_msr, 0);
1769                 break;
1770         case KVM_TPR_ACCESS_REPORTING: {
1771                 struct kvm_tpr_access_ctl tac;
1772
1773                 r = -EFAULT;
1774                 if (copy_from_user(&tac, argp, sizeof tac))
1775                         goto out;
1776                 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
1777                 if (r)
1778                         goto out;
1779                 r = -EFAULT;
1780                 if (copy_to_user(argp, &tac, sizeof tac))
1781                         goto out;
1782                 r = 0;
1783                 break;
1784         };
1785         case KVM_SET_VAPIC_ADDR: {
1786                 struct kvm_vapic_addr va;
1787
1788                 r = -EINVAL;
1789                 if (!irqchip_in_kernel(vcpu->kvm))
1790                         goto out;
1791                 r = -EFAULT;
1792                 if (copy_from_user(&va, argp, sizeof va))
1793                         goto out;
1794                 r = 0;
1795                 kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
1796                 break;
1797         }
1798         case KVM_X86_SETUP_MCE: {
1799                 u64 mcg_cap;
1800
1801                 r = -EFAULT;
1802                 if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
1803                         goto out;
1804                 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
1805                 break;
1806         }
1807         case KVM_X86_SET_MCE: {
1808                 struct kvm_x86_mce mce;
1809
1810                 r = -EFAULT;
1811                 if (copy_from_user(&mce, argp, sizeof mce))
1812                         goto out;
1813                 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
1814                 break;
1815         }
1816         default:
1817                 r = -EINVAL;
1818         }
1819 out:
1820         kfree(lapic);
1821         return r;
1822 }
1823
1824 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
1825 {
1826         int ret;
1827
1828         if (addr > (unsigned int)(-3 * PAGE_SIZE))
1829                 return -1;
1830         ret = kvm_x86_ops->set_tss_addr(kvm, addr);
1831         return ret;
1832 }
1833
1834 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
1835                                           u32 kvm_nr_mmu_pages)
1836 {
1837         if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
1838                 return -EINVAL;
1839
1840         down_write(&kvm->slots_lock);
1841         spin_lock(&kvm->mmu_lock);
1842
1843         kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
1844         kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
1845
1846         spin_unlock(&kvm->mmu_lock);
1847         up_write(&kvm->slots_lock);
1848         return 0;
1849 }
1850
1851 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
1852 {
1853         return kvm->arch.n_alloc_mmu_pages;
1854 }
1855
1856 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
1857 {
1858         int i;
1859         struct kvm_mem_alias *alias;
1860
1861         for (i = 0; i < kvm->arch.naliases; ++i) {
1862                 alias = &kvm->arch.aliases[i];
1863                 if (gfn >= alias->base_gfn
1864                     && gfn < alias->base_gfn + alias->npages)
1865                         return alias->target_gfn + gfn - alias->base_gfn;
1866         }
1867         return gfn;
1868 }
1869
1870 /*
1871  * Set a new alias region.  Aliases map a portion of physical memory into
1872  * another portion.  This is useful for memory windows, for example the PC
1873  * VGA region.
1874  */
1875 static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
1876                                          struct kvm_memory_alias *alias)
1877 {
1878         int r, n;
1879         struct kvm_mem_alias *p;
1880
1881         r = -EINVAL;
1882         /* General sanity checks */
1883         if (alias->memory_size & (PAGE_SIZE - 1))
1884                 goto out;
1885         if (alias->guest_phys_addr & (PAGE_SIZE - 1))
1886                 goto out;
1887         if (alias->slot >= KVM_ALIAS_SLOTS)
1888                 goto out;
1889         if (alias->guest_phys_addr + alias->memory_size
1890             < alias->guest_phys_addr)
1891                 goto out;
1892         if (alias->target_phys_addr + alias->memory_size
1893             < alias->target_phys_addr)
1894                 goto out;
1895
1896         down_write(&kvm->slots_lock);
1897         spin_lock(&kvm->mmu_lock);
1898
1899         p = &kvm->arch.aliases[alias->slot];
1900         p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
1901         p->npages = alias->memory_size >> PAGE_SHIFT;
1902         p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
1903
1904         for (n = KVM_ALIAS_SLOTS; n > 0; --n)
1905                 if (kvm->arch.aliases[n - 1].npages)
1906                         break;
1907         kvm->arch.naliases = n;
1908
1909         spin_unlock(&kvm->mmu_lock);
1910         kvm_mmu_zap_all(kvm);
1911
1912         up_write(&kvm->slots_lock);
1913
1914         return 0;
1915
1916 out:
1917         return r;
1918 }
1919
1920 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
1921 {
1922         int r;
1923
1924         r = 0;
1925         switch (chip->chip_id) {
1926         case KVM_IRQCHIP_PIC_MASTER:
1927                 memcpy(&chip->chip.pic,
1928                         &pic_irqchip(kvm)->pics[0],
1929                         sizeof(struct kvm_pic_state));
1930                 break;
1931         case KVM_IRQCHIP_PIC_SLAVE:
1932                 memcpy(&chip->chip.pic,
1933                         &pic_irqchip(kvm)->pics[1],
1934                         sizeof(struct kvm_pic_state));
1935                 break;
1936         case KVM_IRQCHIP_IOAPIC:
1937                 memcpy(&chip->chip.ioapic,
1938                         ioapic_irqchip(kvm),
1939                         sizeof(struct kvm_ioapic_state));
1940                 break;
1941         default:
1942                 r = -EINVAL;
1943                 break;
1944         }
1945         return r;
1946 }
1947
1948 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
1949 {
1950         int r;
1951
1952         r = 0;
1953         switch (chip->chip_id) {
1954         case KVM_IRQCHIP_PIC_MASTER:
1955                 memcpy(&pic_irqchip(kvm)->pics[0],
1956                         &chip->chip.pic,
1957                         sizeof(struct kvm_pic_state));
1958                 break;
1959         case KVM_IRQCHIP_PIC_SLAVE:
1960                 memcpy(&pic_irqchip(kvm)->pics[1],
1961                         &chip->chip.pic,
1962                         sizeof(struct kvm_pic_state));
1963                 break;
1964         case KVM_IRQCHIP_IOAPIC:
1965                 memcpy(ioapic_irqchip(kvm),
1966                         &chip->chip.ioapic,
1967                         sizeof(struct kvm_ioapic_state));
1968                 break;
1969         default:
1970                 r = -EINVAL;
1971                 break;
1972         }
1973         kvm_pic_update_irq(pic_irqchip(kvm));
1974         return r;
1975 }
1976
1977 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
1978 {
1979         int r = 0;
1980
1981         memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
1982         return r;
1983 }
1984
1985 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
1986 {
1987         int r = 0;
1988
1989         memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
1990         kvm_pit_load_count(kvm, 0, ps->channels[0].count);
1991         return r;
1992 }
1993
1994 static int kvm_vm_ioctl_reinject(struct kvm *kvm,
1995                                  struct kvm_reinject_control *control)
1996 {
1997         if (!kvm->arch.vpit)
1998                 return -ENXIO;
1999         kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
2000         return 0;
2001 }
2002
2003 /*
2004  * Get (and clear) the dirty memory log for a memory slot.
2005  */
2006 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
2007                                       struct kvm_dirty_log *log)
2008 {
2009         int r;
2010         int n;
2011         struct kvm_memory_slot *memslot;
2012         int is_dirty = 0;
2013
2014         down_write(&kvm->slots_lock);
2015
2016         r = kvm_get_dirty_log(kvm, log, &is_dirty);
2017         if (r)
2018                 goto out;
2019
2020         /* If nothing is dirty, don't bother messing with page tables. */
2021         if (is_dirty) {
2022                 spin_lock(&kvm->mmu_lock);
2023                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
2024                 spin_unlock(&kvm->mmu_lock);
2025                 kvm_flush_remote_tlbs(kvm);
2026                 memslot = &kvm->memslots[log->slot];
2027                 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
2028                 memset(memslot->dirty_bitmap, 0, n);
2029         }
2030         r = 0;
2031 out:
2032         up_write(&kvm->slots_lock);
2033         return r;
2034 }
2035
2036 long kvm_arch_vm_ioctl(struct file *filp,
2037                        unsigned int ioctl, unsigned long arg)
2038 {
2039         struct kvm *kvm = filp->private_data;
2040         void __user *argp = (void __user *)arg;
2041         int r = -EINVAL;
2042         /*
2043          * This union makes it completely explicit to gcc-3.x
2044          * that these two variables' stack usage should be
2045          * combined, not added together.
2046          */
2047         union {
2048                 struct kvm_pit_state ps;
2049                 struct kvm_memory_alias alias;
2050                 struct kvm_pit_config pit_config;
2051         } u;
2052
2053         switch (ioctl) {
2054         case KVM_SET_TSS_ADDR:
2055                 r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
2056                 if (r < 0)
2057                         goto out;
2058                 break;
2059         case KVM_SET_MEMORY_REGION: {
2060                 struct kvm_memory_region kvm_mem;
2061                 struct kvm_userspace_memory_region kvm_userspace_mem;
2062
2063                 r = -EFAULT;
2064                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2065                         goto out;
2066                 kvm_userspace_mem.slot = kvm_mem.slot;
2067                 kvm_userspace_mem.flags = kvm_mem.flags;
2068                 kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
2069                 kvm_userspace_mem.memory_size = kvm_mem.memory_size;
2070                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
2071                 if (r)
2072                         goto out;
2073                 break;
2074         }
2075         case KVM_SET_NR_MMU_PAGES:
2076                 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
2077                 if (r)
2078                         goto out;
2079                 break;
2080         case KVM_GET_NR_MMU_PAGES:
2081                 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
2082                 break;
2083         case KVM_SET_MEMORY_ALIAS:
2084                 r = -EFAULT;
2085                 if (copy_from_user(&u.alias, argp, sizeof(struct kvm_memory_alias)))
2086                         goto out;
2087                 r = kvm_vm_ioctl_set_memory_alias(kvm, &u.alias);
2088                 if (r)
2089                         goto out;
2090                 break;
2091         case KVM_CREATE_IRQCHIP:
2092                 r = -ENOMEM;
2093                 kvm->arch.vpic = kvm_create_pic(kvm);
2094                 if (kvm->arch.vpic) {
2095                         r = kvm_ioapic_init(kvm);
2096                         if (r) {
2097                                 kfree(kvm->arch.vpic);
2098                                 kvm->arch.vpic = NULL;
2099                                 goto out;
2100                         }
2101                 } else
2102                         goto out;
2103                 r = kvm_setup_default_irq_routing(kvm);
2104                 if (r) {
2105                         kfree(kvm->arch.vpic);
2106                         kfree(kvm->arch.vioapic);
2107                         goto out;
2108                 }
2109                 break;
2110         case KVM_CREATE_PIT:
2111                 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
2112                 goto create_pit;
2113         case KVM_CREATE_PIT2:
2114                 r = -EFAULT;
2115                 if (copy_from_user(&u.pit_config, argp,
2116                                    sizeof(struct kvm_pit_config)))
2117                         goto out;
2118         create_pit:
2119                 mutex_lock(&kvm->lock);
2120                 r = -EEXIST;
2121                 if (kvm->arch.vpit)
2122                         goto create_pit_unlock;
2123                 r = -ENOMEM;
2124                 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
2125                 if (kvm->arch.vpit)
2126                         r = 0;
2127         create_pit_unlock:
2128                 mutex_unlock(&kvm->lock);
2129                 break;
2130         case KVM_IRQ_LINE_STATUS:
2131         case KVM_IRQ_LINE: {
2132                 struct kvm_irq_level irq_event;
2133
2134                 r = -EFAULT;
2135                 if (copy_from_user(&irq_event, argp, sizeof irq_event))
2136                         goto out;
2137                 if (irqchip_in_kernel(kvm)) {
2138                         __s32 status;
2139                         mutex_lock(&kvm->irq_lock);
2140                         status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
2141                                         irq_event.irq, irq_event.level);
2142                         mutex_unlock(&kvm->irq_lock);
2143                         if (ioctl == KVM_IRQ_LINE_STATUS) {
2144                                 irq_event.status = status;
2145                                 if (copy_to_user(argp, &irq_event,
2146                                                         sizeof irq_event))
2147                                         goto out;
2148                         }
2149                         r = 0;
2150                 }
2151                 break;
2152         }
2153         case KVM_GET_IRQCHIP: {
2154                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
2155                 struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
2156
2157                 r = -ENOMEM;
2158                 if (!chip)
2159                         goto out;
2160                 r = -EFAULT;
2161                 if (copy_from_user(chip, argp, sizeof *chip))
2162                         goto get_irqchip_out;
2163                 r = -ENXIO;
2164                 if (!irqchip_in_kernel(kvm))
2165                         goto get_irqchip_out;
2166                 r = kvm_vm_ioctl_get_irqchip(kvm, chip);
2167                 if (r)
2168                         goto get_irqchip_out;
2169                 r = -EFAULT;
2170                 if (copy_to_user(argp, chip, sizeof *chip))
2171                         goto get_irqchip_out;
2172                 r = 0;
2173         get_irqchip_out:
2174                 kfree(chip);
2175                 if (r)
2176                         goto out;
2177                 break;
2178         }
2179         case KVM_SET_IRQCHIP: {
2180                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
2181                 struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
2182
2183                 r = -ENOMEM;
2184                 if (!chip)
2185                         goto out;
2186                 r = -EFAULT;
2187                 if (copy_from_user(chip, argp, sizeof *chip))
2188                         goto set_irqchip_out;
2189                 r = -ENXIO;
2190                 if (!irqchip_in_kernel(kvm))
2191                         goto set_irqchip_out;
2192                 r = kvm_vm_ioctl_set_irqchip(kvm, chip);
2193                 if (r)
2194                         goto set_irqchip_out;
2195                 r = 0;
2196         set_irqchip_out:
2197                 kfree(chip);
2198                 if (r)
2199                         goto out;
2200                 break;
2201         }
2202         case KVM_GET_PIT: {
2203                 r = -EFAULT;
2204                 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
2205                         goto out;
2206                 r = -ENXIO;
2207                 if (!kvm->arch.vpit)
2208                         goto out;
2209                 r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
2210                 if (r)
2211                         goto out;
2212                 r = -EFAULT;
2213                 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
2214                         goto out;
2215                 r = 0;
2216                 break;
2217         }
2218         case KVM_SET_PIT: {
2219                 r = -EFAULT;
2220                 if (copy_from_user(&u.ps, argp, sizeof u.ps))
2221                         goto out;
2222                 r = -ENXIO;
2223                 if (!kvm->arch.vpit)
2224                         goto out;
2225                 r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
2226                 if (r)
2227                         goto out;
2228                 r = 0;
2229                 break;
2230         }
2231         case KVM_REINJECT_CONTROL: {
2232                 struct kvm_reinject_control control;
2233                 r =  -EFAULT;
2234                 if (copy_from_user(&control, argp, sizeof(control)))
2235                         goto out;
2236                 r = kvm_vm_ioctl_reinject(kvm, &control);
2237                 if (r)
2238                         goto out;
2239                 r = 0;
2240                 break;
2241         }
2242         default:
2243                 ;
2244         }
2245 out:
2246         return r;
2247 }
2248
2249 static void kvm_init_msr_list(void)
2250 {
2251         u32 dummy[2];
2252         unsigned i, j;
2253
2254         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
2255                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
2256                         continue;
2257                 if (j < i)
2258                         msrs_to_save[j] = msrs_to_save[i];
2259                 j++;
2260         }
2261         num_msrs_to_save = j;
2262 }
2263
2264 /*
2265  * Only apic need an MMIO device hook, so shortcut now..
2266  */
2267 static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
2268                                                 gpa_t addr, int len,
2269                                                 int is_write)
2270 {
2271         struct kvm_io_device *dev;
2272
2273         if (vcpu->arch.apic) {
2274                 dev = &vcpu->arch.apic->dev;
2275                 if (kvm_iodevice_in_range(dev, addr, len, is_write))
2276                         return dev;
2277         }
2278         return NULL;
2279 }
2280
2281
2282 static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
2283                                                 gpa_t addr, int len,
2284                                                 int is_write)
2285 {
2286         struct kvm_io_device *dev;
2287
2288         dev = vcpu_find_pervcpu_dev(vcpu, addr, len, is_write);
2289         if (dev == NULL)
2290                 dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr, len,
2291                                           is_write);
2292         return dev;
2293 }
2294
2295 static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
2296                                struct kvm_vcpu *vcpu)
2297 {
2298         void *data = val;
2299         int r = X86EMUL_CONTINUE;
2300
2301         while (bytes) {
2302                 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
2303                 unsigned offset = addr & (PAGE_SIZE-1);
2304                 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
2305                 int ret;
2306
2307                 if (gpa == UNMAPPED_GVA) {
2308                         r = X86EMUL_PROPAGATE_FAULT;
2309                         goto out;
2310                 }
2311                 ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
2312                 if (ret < 0) {
2313                         r = X86EMUL_UNHANDLEABLE;
2314                         goto out;
2315                 }
2316
2317                 bytes -= toread;
2318                 data += toread;
2319                 addr += toread;
2320         }
2321 out:
2322         return r;
2323 }
2324
2325 static int kvm_write_guest_virt(gva_t addr, void *val, unsigned int bytes,
2326                                 struct kvm_vcpu *vcpu)
2327 {
2328         void *data = val;
2329         int r = X86EMUL_CONTINUE;
2330
2331         while (bytes) {
2332                 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
2333                 unsigned offset = addr & (PAGE_SIZE-1);
2334                 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
2335                 int ret;
2336
2337                 if (gpa == UNMAPPED_GVA) {
2338                         r = X86EMUL_PROPAGATE_FAULT;
2339                         goto out;
2340                 }
2341                 ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
2342                 if (ret < 0) {
2343                         r = X86EMUL_UNHANDLEABLE;
2344                         goto out;
2345                 }
2346
2347                 bytes -= towrite;
2348                 data += towrite;
2349                 addr += towrite;
2350         }
2351 out:
2352         return r;
2353 }
2354
2355
2356 static int emulator_read_emulated(unsigned long addr,
2357                                   void *val,
2358                                   unsigned int bytes,
2359                                   struct kvm_vcpu *vcpu)
2360 {
2361         struct kvm_io_device *mmio_dev;
2362         gpa_t                 gpa;
2363
2364         if (vcpu->mmio_read_completed) {
2365                 memcpy(val, vcpu->mmio_data, bytes);
2366                 vcpu->mmio_read_completed = 0;
2367                 return X86EMUL_CONTINUE;
2368         }
2369
2370         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
2371
2372         /* For APIC access vmexit */
2373         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
2374                 goto mmio;
2375
2376         if (kvm_read_guest_virt(addr, val, bytes, vcpu)
2377                                 == X86EMUL_CONTINUE)
2378                 return X86EMUL_CONTINUE;
2379         if (gpa == UNMAPPED_GVA)
2380                 return X86EMUL_PROPAGATE_FAULT;
2381
2382 mmio:
2383         /*
2384          * Is this MMIO handled locally?
2385          */
2386         mutex_lock(&vcpu->kvm->lock);
2387         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 0);
2388         mutex_unlock(&vcpu->kvm->lock);
2389         if (mmio_dev) {
2390                 kvm_iodevice_read(mmio_dev, gpa, bytes, val);
2391                 return X86EMUL_CONTINUE;
2392         }
2393
2394         vcpu->mmio_needed = 1;
2395         vcpu->mmio_phys_addr = gpa;
2396         vcpu->mmio_size = bytes;
2397         vcpu->mmio_is_write = 0;
2398
2399         return X86EMUL_UNHANDLEABLE;
2400 }
2401
2402 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
2403                           const void *val, int bytes)
2404 {
2405         int ret;
2406
2407         ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
2408         if (ret < 0)
2409                 return 0;
2410         kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
2411         return 1;
2412 }
2413
2414 static int emulator_write_emulated_onepage(unsigned long addr,
2415                                            const void *val,
2416                                            unsigned int bytes,
2417                                            struct kvm_vcpu *vcpu)
2418 {
2419         struct kvm_io_device *mmio_dev;
2420         gpa_t                 gpa;
2421
2422         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
2423
2424         if (gpa == UNMAPPED_GVA) {
2425                 kvm_inject_page_fault(vcpu, addr, 2);
2426                 return X86EMUL_PROPAGATE_FAULT;
2427         }
2428
2429         /* For APIC access vmexit */
2430         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
2431                 goto mmio;
2432
2433         if (emulator_write_phys(vcpu, gpa, val, bytes))
2434                 return X86EMUL_CONTINUE;
2435
2436 mmio:
2437         /*
2438          * Is this MMIO handled locally?
2439          */
2440         mutex_lock(&vcpu->kvm->lock);
2441         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 1);
2442         mutex_unlock(&vcpu->kvm->lock);
2443         if (mmio_dev) {
2444                 kvm_iodevice_write(mmio_dev, gpa, bytes, val);
2445                 return X86EMUL_CONTINUE;
2446         }
2447
2448         vcpu->mmio_needed = 1;
2449         vcpu->mmio_phys_addr = gpa;
2450         vcpu->mmio_size = bytes;
2451         vcpu->mmio_is_write = 1;
2452         memcpy(vcpu->mmio_data, val, bytes);
2453
2454         return X86EMUL_CONTINUE;
2455 }
2456
2457 int emulator_write_emulated(unsigned long addr,
2458                                    const void *val,
2459                                    unsigned int bytes,
2460                                    struct kvm_vcpu *vcpu)
2461 {
2462         /* Crossing a page boundary? */
2463         if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
2464                 int rc, now;
2465
2466                 now = -addr & ~PAGE_MASK;
2467                 rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
2468                 if (rc != X86EMUL_CONTINUE)
2469                         return rc;
2470                 addr += now;
2471                 val += now;
2472                 bytes -= now;
2473         }
2474         return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
2475 }
2476 EXPORT_SYMBOL_GPL(emulator_write_emulated);
2477
2478 static int emulator_cmpxchg_emulated(unsigned long addr,
2479                                      const void *old,
2480                                      const void *new,
2481                                      unsigned int bytes,
2482                                      struct kvm_vcpu *vcpu)
2483 {
2484         static int reported;
2485
2486         if (!reported) {
2487                 reported = 1;
2488                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
2489         }
2490 #ifndef CONFIG_X86_64
2491         /* guests cmpxchg8b have to be emulated atomically */
2492         if (bytes == 8) {
2493                 gpa_t gpa;
2494                 struct page *page;
2495                 char *kaddr;
2496                 u64 val;
2497
2498                 gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
2499
2500                 if (gpa == UNMAPPED_GVA ||
2501                    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
2502                         goto emul_write;
2503
2504                 if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
2505                         goto emul_write;
2506
2507                 val = *(u64 *)new;
2508
2509                 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2510
2511                 kaddr = kmap_atomic(page, KM_USER0);
2512                 set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val);
2513                 kunmap_atomic(kaddr, KM_USER0);
2514                 kvm_release_page_dirty(page);
2515         }
2516 emul_write:
2517 #endif
2518
2519         return emulator_write_emulated(addr, new, bytes, vcpu);
2520 }
2521
2522 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
2523 {
2524         return kvm_x86_ops->get_segment_base(vcpu, seg);
2525 }
2526
2527 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
2528 {
2529         kvm_mmu_invlpg(vcpu, address);
2530         return X86EMUL_CONTINUE;
2531 }
2532
2533 int emulate_clts(struct kvm_vcpu *vcpu)
2534 {
2535         KVMTRACE_0D(CLTS, vcpu, handler);
2536         kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS);
2537         return X86EMUL_CONTINUE;
2538 }
2539
2540 int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
2541 {
2542         struct kvm_vcpu *vcpu = ctxt->vcpu;
2543
2544         switch (dr) {
2545         case 0 ... 3:
2546                 *dest = kvm_x86_ops->get_dr(vcpu, dr);
2547                 return X86EMUL_CONTINUE;
2548         default:
2549                 pr_unimpl(vcpu, "%s: unexpected dr %u\n", __func__, dr);
2550                 return X86EMUL_UNHANDLEABLE;
2551         }
2552 }
2553
2554 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
2555 {
2556         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
2557         int exception;
2558
2559         kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
2560         if (exception) {
2561                 /* FIXME: better handling */
2562                 return X86EMUL_UNHANDLEABLE;
2563         }
2564         return X86EMUL_CONTINUE;
2565 }
2566
2567 void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
2568 {
2569         u8 opcodes[4];
2570         unsigned long rip = kvm_rip_read(vcpu);
2571         unsigned long rip_linear;
2572
2573         if (!printk_ratelimit())
2574                 return;
2575
2576         rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
2577
2578         kvm_read_guest_virt(rip_linear, (void *)opcodes, 4, vcpu);
2579
2580         printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
2581                context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
2582 }
2583 EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
2584
2585 static struct x86_emulate_ops emulate_ops = {
2586         .read_std            = kvm_read_guest_virt,
2587         .read_emulated       = emulator_read_emulated,
2588         .write_emulated      = emulator_write_emulated,
2589         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
2590 };
2591
2592 static void cache_all_regs(struct kvm_vcpu *vcpu)
2593 {
2594         kvm_register_read(vcpu, VCPU_REGS_RAX);
2595         kvm_register_read(vcpu, VCPU_REGS_RSP);
2596         kvm_register_read(vcpu, VCPU_REGS_RIP);
2597         vcpu->arch.regs_dirty = ~0;
2598 }
2599
2600 int emulate_instruction(struct kvm_vcpu *vcpu,
2601                         struct kvm_run *run,
2602                         unsigned long cr2,
2603                         u16 error_code,
2604                         int emulation_type)
2605 {
2606         int r, shadow_mask;
2607         struct decode_cache *c;
2608
2609         kvm_clear_exception_queue(vcpu);
2610         vcpu->arch.mmio_fault_cr2 = cr2;
2611         /*
2612          * TODO: fix x86_emulate.c to use guest_read/write_register
2613          * instead of direct ->regs accesses, can save hundred cycles
2614          * on Intel for instructions that don't read/change RSP, for
2615          * for example.
2616          */
2617         cache_all_regs(vcpu);
2618
2619         vcpu->mmio_is_write = 0;
2620         vcpu->arch.pio.string = 0;
2621
2622         if (!(emulation_type & EMULTYPE_NO_DECODE)) {
2623                 int cs_db, cs_l;
2624                 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
2625
2626                 vcpu->arch.emulate_ctxt.vcpu = vcpu;
2627                 vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
2628                 vcpu->arch.emulate_ctxt.mode =
2629                         (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
2630                         ? X86EMUL_MODE_REAL : cs_l
2631                         ? X86EMUL_MODE_PROT64 : cs_db
2632                         ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
2633
2634                 r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
2635
2636                 /* Reject the instructions other than VMCALL/VMMCALL when
2637                  * try to emulate invalid opcode */
2638                 c = &vcpu->arch.emulate_ctxt.decode;
2639                 if ((emulation_type & EMULTYPE_TRAP_UD) &&
2640                     (!(c->twobyte && c->b == 0x01 &&
2641                       (c->modrm_reg == 0 || c->modrm_reg == 3) &&
2642                        c->modrm_mod == 3 && c->modrm_rm == 1)))
2643                         return EMULATE_FAIL;
2644
2645                 ++vcpu->stat.insn_emulation;
2646                 if (r)  {
2647                         ++vcpu->stat.insn_emulation_fail;
2648                         if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
2649                                 return EMULATE_DONE;
2650                         return EMULATE_FAIL;
2651                 }
2652         }
2653
2654         if (emulation_type & EMULTYPE_SKIP) {
2655                 kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.decode.eip);
2656                 return EMULATE_DONE;
2657         }
2658
2659         r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
2660         shadow_mask = vcpu->arch.emulate_ctxt.interruptibility;
2661
2662         if (r == 0)
2663                 kvm_x86_ops->set_interrupt_shadow(vcpu, shadow_mask);
2664
2665         if (vcpu->arch.pio.string)
2666                 return EMULATE_DO_MMIO;
2667
2668         if ((r || vcpu->mmio_is_write) && run) {
2669                 run->exit_reason = KVM_EXIT_MMIO;
2670                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
2671                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
2672                 run->mmio.len = vcpu->mmio_size;
2673                 run->mmio.is_write = vcpu->mmio_is_write;
2674         }
2675
2676         if (r) {
2677                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
2678                         return EMULATE_DONE;
2679                 if (!vcpu->mmio_needed) {
2680                         kvm_report_emulation_failure(vcpu, "mmio");
2681                         return EMULATE_FAIL;
2682                 }
2683                 return EMULATE_DO_MMIO;
2684         }
2685
2686         kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
2687
2688         if (vcpu->mmio_is_write) {
2689                 vcpu->mmio_needed = 0;
2690                 return EMULATE_DO_MMIO;
2691         }
2692
2693         return EMULATE_DONE;
2694 }
2695 EXPORT_SYMBOL_GPL(emulate_instruction);
2696
2697 static int pio_copy_data(struct kvm_vcpu *vcpu)
2698 {
2699         void *p = vcpu->arch.pio_data;
2700         gva_t q = vcpu->arch.pio.guest_gva;
2701         unsigned bytes;
2702         int ret;
2703
2704         bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count;
2705         if (vcpu->arch.pio.in)
2706                 ret = kvm_write_guest_virt(q, p, bytes, vcpu);
2707         else
2708                 ret = kvm_read_guest_virt(q, p, bytes, vcpu);
2709         return ret;
2710 }
2711
2712 int complete_pio(struct kvm_vcpu *vcpu)
2713 {
2714         struct kvm_pio_request *io = &vcpu->arch.pio;
2715         long delta;
2716         int r;
2717         unsigned long val;
2718
2719         if (!io->string) {
2720                 if (io->in) {
2721                         val = kvm_register_read(vcpu, VCPU_REGS_RAX);
2722                         memcpy(&val, vcpu->arch.pio_data, io->size);
2723                         kvm_register_write(vcpu, VCPU_REGS_RAX, val);
2724                 }
2725         } else {
2726                 if (io->in) {
2727                         r = pio_copy_data(vcpu);
2728                         if (r)
2729                                 return r;
2730                 }
2731
2732                 delta = 1;
2733                 if (io->rep) {
2734                         delta *= io->cur_count;
2735                         /*
2736                          * The size of the register should really depend on
2737                          * current address size.
2738                          */
2739                         val = kvm_register_read(vcpu, VCPU_REGS_RCX);
2740                         val -= delta;
2741                         kvm_register_write(vcpu, VCPU_REGS_RCX, val);
2742                 }
2743                 if (io->down)
2744                         delta = -delta;
2745                 delta *= io->size;
2746                 if (io->in) {
2747                         val = kvm_register_read(vcpu, VCPU_REGS_RDI);
2748                         val += delta;
2749                         kvm_register_write(vcpu, VCPU_REGS_RDI, val);
2750                 } else {
2751                         val = kvm_register_read(vcpu, VCPU_REGS_RSI);
2752                         val += delta;
2753                         kvm_register_write(vcpu, VCPU_REGS_RSI, val);
2754                 }
2755         }
2756
2757         io->count -= io->cur_count;
2758         io->cur_count = 0;
2759
2760         return 0;
2761 }
2762
2763 static void kernel_pio(struct kvm_io_device *pio_dev,
2764                        struct kvm_vcpu *vcpu,
2765                        void *pd)
2766 {
2767         /* TODO: String I/O for in kernel device */
2768
2769         if (vcpu->arch.pio.in)
2770                 kvm_iodevice_read(pio_dev, vcpu->arch.pio.port,
2771                                   vcpu->arch.pio.size,
2772                                   pd);
2773         else
2774                 kvm_iodevice_write(pio_dev, vcpu->arch.pio.port,
2775                                    vcpu->arch.pio.size,
2776                                    pd);
2777 }
2778
2779 static void pio_string_write(struct kvm_io_device *pio_dev,
2780                              struct kvm_vcpu *vcpu)
2781 {
2782         struct kvm_pio_request *io = &vcpu->arch.pio;
2783         void *pd = vcpu->arch.pio_data;
2784         int i;
2785
2786         for (i = 0; i < io->cur_count; i++) {
2787                 kvm_iodevice_write(pio_dev, io->port,
2788                                    io->size,
2789                                    pd);
2790                 pd += io->size;
2791         }
2792 }
2793
2794 static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
2795                                                gpa_t addr, int len,
2796                                                int is_write)
2797 {
2798         return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr, len, is_write);
2799 }
2800
2801 int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
2802                   int size, unsigned port)
2803 {
2804         struct kvm_io_device *pio_dev;
2805         unsigned long val;
2806
2807         vcpu->run->exit_reason = KVM_EXIT_IO;
2808         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
2809         vcpu->run->io.size = vcpu->arch.pio.size = size;
2810         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
2811         vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1;
2812         vcpu->run->io.port = vcpu->arch.pio.port = port;
2813         vcpu->arch.pio.in = in;
2814         vcpu->arch.pio.string = 0;
2815         vcpu->arch.pio.down = 0;
2816         vcpu->arch.pio.rep = 0;
2817
2818         if (vcpu->run->io.direction == KVM_EXIT_IO_IN)
2819                 KVMTRACE_2D(IO_READ, vcpu, vcpu->run->io.port, (u32)size,
2820                             handler);
2821         else
2822                 KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size,
2823                             handler);
2824
2825         val = kvm_register_read(vcpu, VCPU_REGS_RAX);
2826         memcpy(vcpu->arch.pio_data, &val, 4);
2827
2828         mutex_lock(&vcpu->kvm->lock);
2829         pio_dev = vcpu_find_pio_dev(vcpu, port, size, !in);
2830         mutex_unlock(&vcpu->kvm->lock);
2831         if (pio_dev) {
2832                 kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data);
2833                 complete_pio(vcpu);
2834                 return 1;
2835         }
2836         return 0;
2837 }
2838 EXPORT_SYMBOL_GPL(kvm_emulate_pio);
2839
2840 int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
2841                   int size, unsigned long count, int down,
2842                   gva_t address, int rep, unsigned port)
2843 {
2844         unsigned now, in_page;
2845         int ret = 0;
2846         struct kvm_io_device *pio_dev;
2847
2848         vcpu->run->exit_reason = KVM_EXIT_IO;
2849         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
2850         vcpu->run->io.size = vcpu->arch.pio.size = size;
2851         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
2852         vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count;
2853         vcpu->run->io.port = vcpu->arch.pio.port = port;
2854         vcpu->arch.pio.in = in;
2855         vcpu->arch.pio.string = 1;
2856         vcpu->arch.pio.down = down;
2857         vcpu->arch.pio.rep = rep;
2858
2859         if (vcpu->run->io.direction == KVM_EXIT_IO_IN)
2860                 KVMTRACE_2D(IO_READ, vcpu, vcpu->run->io.port, (u32)size,
2861                             handler);
2862         else
2863                 KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size,
2864                             handler);
2865
2866         if (!count) {
2867                 kvm_x86_ops->skip_emulated_instruction(vcpu);
2868                 return 1;
2869         }
2870
2871         if (!down)
2872                 in_page = PAGE_SIZE - offset_in_page(address);
2873         else
2874                 in_page = offset_in_page(address) + size;
2875         now = min(count, (unsigned long)in_page / size);
2876         if (!now)
2877                 now = 1;
2878         if (down) {
2879                 /*
2880                  * String I/O in reverse.  Yuck.  Kill the guest, fix later.
2881                  */
2882                 pr_unimpl(vcpu, "guest string pio down\n");
2883                 kvm_inject_gp(vcpu, 0);
2884                 return 1;
2885         }
2886         vcpu->run->io.count = now;
2887         vcpu->arch.pio.cur_count = now;
2888
2889         if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count)
2890                 kvm_x86_ops->skip_emulated_instruction(vcpu);
2891
2892         vcpu->arch.pio.guest_gva = address;
2893
2894         mutex_lock(&vcpu->kvm->lock);
2895         pio_dev = vcpu_find_pio_dev(vcpu, port,
2896                                     vcpu->arch.pio.cur_count,
2897                                     !vcpu->arch.pio.in);
2898         mutex_unlock(&vcpu->kvm->lock);
2899
2900         if (!vcpu->arch.pio.in) {
2901                 /* string PIO write */
2902                 ret = pio_copy_data(vcpu);
2903                 if (ret == X86EMUL_PROPAGATE_FAULT) {
2904                         kvm_inject_gp(vcpu, 0);
2905                         return 1;
2906                 }
2907                 if (ret == 0 && pio_dev) {
2908                         pio_string_write(pio_dev, vcpu);
2909                         complete_pio(vcpu);
2910                         if (vcpu->arch.pio.count == 0)
2911                                 ret = 1;
2912                 }
2913         } else if (pio_dev)
2914                 pr_unimpl(vcpu, "no string pio read support yet, "
2915                        "port %x size %d count %ld\n",
2916                         port, size, count);
2917
2918         return ret;
2919 }
2920 EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
2921
2922 static void bounce_off(void *info)
2923 {
2924         /* nothing */
2925 }
2926
2927 static unsigned int  ref_freq;
2928 static unsigned long tsc_khz_ref;
2929
2930 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
2931                                      void *data)
2932 {
2933         struct cpufreq_freqs *freq = data;
2934         struct kvm *kvm;
2935         struct kvm_vcpu *vcpu;
2936         int i, send_ipi = 0;
2937
2938         if (!ref_freq)
2939                 ref_freq = freq->old;
2940
2941         if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
2942                 return 0;
2943         if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
2944                 return 0;
2945         per_cpu(cpu_tsc_khz, freq->cpu) = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
2946
2947         spin_lock(&kvm_lock);
2948         list_for_each_entry(kvm, &vm_list, vm_list) {
2949                 kvm_for_each_vcpu(i, vcpu, kvm) {
2950                         if (vcpu->cpu != freq->cpu)
2951                                 continue;
2952                         if (!kvm_request_guest_time_update(vcpu))
2953                                 continue;
2954                         if (vcpu->cpu != smp_processor_id())
2955                                 send_ipi++;
2956                 }
2957         }
2958         spin_unlock(&kvm_lock);
2959
2960         if (freq->old < freq->new && send_ipi) {
2961                 /*
2962                  * We upscale the frequency.  Must make the guest
2963                  * doesn't see old kvmclock values while running with
2964                  * the new frequency, otherwise we risk the guest sees
2965                  * time go backwards.
2966                  *
2967                  * In case we update the frequency for another cpu
2968                  * (which might be in guest context) send an interrupt
2969                  * to kick the cpu out of guest context.  Next time
2970                  * guest context is entered kvmclock will be updated,
2971                  * so the guest will not see stale values.
2972                  */
2973                 smp_call_function_single(freq->cpu, bounce_off, NULL, 1);
2974         }
2975         return 0;
2976 }
2977
2978 static struct notifier_block kvmclock_cpufreq_notifier_block = {
2979         .notifier_call  = kvmclock_cpufreq_notifier
2980 };
2981
2982 int kvm_arch_init(void *opaque)
2983 {
2984         int r, cpu;
2985         struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
2986
2987         if (kvm_x86_ops) {
2988                 printk(KERN_ERR "kvm: already loaded the other module\n");
2989                 r = -EEXIST;
2990                 goto out;
2991         }
2992
2993         if (!ops->cpu_has_kvm_support()) {
2994                 printk(KERN_ERR "kvm: no hardware support\n");
2995                 r = -EOPNOTSUPP;
2996                 goto out;
2997         }
2998         if (ops->disabled_by_bios()) {
2999                 printk(KERN_ERR "kvm: disabled by bios\n");
3000                 r = -EOPNOTSUPP;
3001                 goto out;
3002         }
3003
3004         r = kvm_mmu_module_init();
3005         if (r)
3006                 goto out;
3007
3008         kvm_init_msr_list();
3009
3010         kvm_x86_ops = ops;
3011         kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
3012         kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
3013         kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
3014                         PT_DIRTY_MASK, PT64_NX_MASK, 0);
3015
3016         for_each_possible_cpu(cpu)
3017                 per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
3018         if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
3019                 tsc_khz_ref = tsc_khz;
3020                 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
3021                                           CPUFREQ_TRANSITION_NOTIFIER);
3022         }
3023
3024         return 0;
3025
3026 out:
3027         return r;
3028 }
3029
3030 void kvm_arch_exit(void)
3031 {
3032         if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
3033                 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
3034                                             CPUFREQ_TRANSITION_NOTIFIER);
3035         kvm_x86_ops = NULL;
3036         kvm_mmu_module_exit();
3037 }
3038
3039 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
3040 {
3041         ++vcpu->stat.halt_exits;
3042         KVMTRACE_0D(HLT, vcpu, handler);
3043         if (irqchip_in_kernel(vcpu->kvm)) {
3044                 vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
3045                 return 1;
3046         } else {
3047                 vcpu->run->exit_reason = KVM_EXIT_HLT;
3048                 return 0;
3049         }
3050 }
3051 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
3052
3053 static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
3054                            unsigned long a1)
3055 {
3056         if (is_long_mode(vcpu))
3057                 return a0;
3058         else
3059                 return a0 | ((gpa_t)a1 << 32);
3060 }
3061
3062 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
3063 {
3064         unsigned long nr, a0, a1, a2, a3, ret;
3065         int r = 1;
3066
3067         nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
3068         a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
3069         a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
3070         a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
3071         a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
3072
3073         KVMTRACE_1D(VMMCALL, vcpu, (u32)nr, handler);
3074
3075         if (!is_long_mode(vcpu)) {
3076                 nr &= 0xFFFFFFFF;
3077                 a0 &= 0xFFFFFFFF;
3078                 a1 &= 0xFFFFFFFF;
3079                 a2 &= 0xFFFFFFFF;
3080                 a3 &= 0xFFFFFFFF;
3081         }
3082
3083         switch (nr) {
3084         case KVM_HC_VAPIC_POLL_IRQ:
3085                 ret = 0;
3086                 break;
3087         case KVM_HC_MMU_OP:
3088                 r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
3089                 break;
3090         default:
3091                 ret = -KVM_ENOSYS;
3092                 break;
3093         }
3094         kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
3095         ++vcpu->stat.hypercalls;
3096         return r;
3097 }
3098 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
3099
3100 int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
3101 {
3102         char instruction[3];
3103         int ret = 0;
3104         unsigned long rip = kvm_rip_read(vcpu);
3105
3106
3107         /*
3108          * Blow out the MMU to ensure that no other VCPU has an active mapping
3109          * to ensure that the updated hypercall appears atomically across all
3110          * VCPUs.
3111          */
3112         kvm_mmu_zap_all(vcpu->kvm);
3113
3114         kvm_x86_ops->patch_hypercall(vcpu, instruction);
3115         if (emulator_write_emulated(rip, instruction, 3, vcpu)
3116             != X86EMUL_CONTINUE)
3117                 ret = -EFAULT;
3118
3119         return ret;
3120 }
3121
3122 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
3123 {
3124         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
3125 }
3126
3127 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
3128 {
3129         struct descriptor_table dt = { limit, base };
3130
3131         kvm_x86_ops->set_gdt(vcpu, &dt);
3132 }
3133
3134 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
3135 {
3136         struct descriptor_table dt = { limit, base };
3137
3138         kvm_x86_ops->set_idt(vcpu, &dt);
3139 }
3140
3141 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
3142                    unsigned long *rflags)
3143 {
3144         kvm_lmsw(vcpu, msw);
3145         *rflags = kvm_x86_ops->get_rflags(vcpu);
3146 }
3147
3148 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
3149 {
3150         unsigned long value;
3151
3152         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
3153         switch (cr) {
3154         case 0:
3155                 value = vcpu->arch.cr0;
3156                 break;
3157         case 2:
3158                 value = vcpu->arch.cr2;
3159                 break;
3160         case 3:
3161                 value = vcpu->arch.cr3;
3162                 break;
3163         case 4:
3164                 value = vcpu->arch.cr4;
3165                 break;
3166         case 8:
3167                 value = kvm_get_cr8(vcpu);
3168                 break;
3169         default:
3170                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
3171                 return 0;
3172         }
3173         KVMTRACE_3D(CR_READ, vcpu, (u32)cr, (u32)value,
3174                     (u32)((u64)value >> 32), handler);
3175
3176         return value;
3177 }
3178
3179 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
3180                      unsigned long *rflags)
3181 {
3182         KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr, (u32)val,
3183                     (u32)((u64)val >> 32), handler);
3184
3185         switch (cr) {
3186         case 0:
3187                 kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val));
3188                 *rflags = kvm_x86_ops->get_rflags(vcpu);
3189                 break;
3190         case 2:
3191                 vcpu->arch.cr2 = val;
3192                 break;
3193         case 3:
3194                 kvm_set_cr3(vcpu, val);
3195                 break;
3196         case 4:
3197                 kvm_set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val));
3198                 break;
3199         case 8:
3200                 kvm_set_cr8(vcpu, val & 0xfUL);
3201                 break;
3202         default:
3203                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
3204         }
3205 }
3206
3207 static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
3208 {
3209         struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
3210         int j, nent = vcpu->arch.cpuid_nent;
3211
3212         e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
3213         /* when no next entry is found, the current entry[i] is reselected */
3214         for (j = i + 1; ; j = (j + 1) % nent) {
3215                 struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
3216                 if (ej->function == e->function) {
3217                         ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
3218                         return j;
3219                 }
3220         }
3221         return 0; /* silence gcc, even though control never reaches here */
3222 }
3223
3224 /* find an entry with matching function, matching index (if needed), and that
3225  * should be read next (if it's stateful) */
3226 static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
3227         u32 function, u32 index)
3228 {
3229         if (e->function != function)
3230                 return 0;
3231         if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
3232                 return 0;
3233         if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
3234             !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
3235                 return 0;
3236         return 1;
3237 }
3238
3239 struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
3240                                               u32 function, u32 index)
3241 {
3242         int i;
3243         struct kvm_cpuid_entry2 *best = NULL;
3244
3245         for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
3246                 struct kvm_cpuid_entry2 *e;
3247
3248                 e = &vcpu->arch.cpuid_entries[i];
3249                 if (is_matching_cpuid_entry(e, function, index)) {
3250                         if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
3251                                 move_to_next_stateful_cpuid_entry(vcpu, i);
3252                         best = e;
3253                         break;
3254                 }
3255                 /*
3256                  * Both basic or both extended?
3257                  */
3258                 if (((e->function ^ function) & 0x80000000) == 0)
3259                         if (!best || e->function > best->function)
3260                                 best = e;
3261         }
3262         return best;
3263 }
3264
3265 int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
3266 {
3267         struct kvm_cpuid_entry2 *best;
3268
3269         best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
3270         if (best)
3271                 return best->eax & 0xff;
3272         return 36;
3273 }
3274
3275 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
3276 {
3277         u32 function, index;
3278         struct kvm_cpuid_entry2 *best;
3279
3280         function = kvm_register_read(vcpu, VCPU_REGS_RAX);
3281         index = kvm_register_read(vcpu, VCPU_REGS_RCX);
3282         kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
3283         kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
3284         kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
3285         kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
3286         best = kvm_find_cpuid_entry(vcpu, function, index);
3287         if (best) {
3288                 kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
3289                 kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
3290                 kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
3291                 kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
3292         }
3293         kvm_x86_ops->skip_emulated_instruction(vcpu);
3294         KVMTRACE_5D(CPUID, vcpu, function,
3295                     (u32)kvm_register_read(vcpu, VCPU_REGS_RAX),
3296                     (u32)kvm_register_read(vcpu, VCPU_REGS_RBX),
3297                     (u32)kvm_register_read(vcpu, VCPU_REGS_RCX),
3298                     (u32)kvm_register_read(vcpu, VCPU_REGS_RDX), handler);
3299 }
3300 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
3301
3302 /*
3303  * Check if userspace requested an interrupt window, and that the
3304  * interrupt window is open.
3305  *
3306  * No need to exit to userspace if we already have an interrupt queued.
3307  */
3308 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
3309                                           struct kvm_run *kvm_run)
3310 {
3311         return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
3312                 kvm_run->request_interrupt_window &&
3313                 kvm_arch_interrupt_allowed(vcpu));
3314 }
3315
3316 static void post_kvm_run_save(struct kvm_vcpu *vcpu,
3317                               struct kvm_run *kvm_run)
3318 {
3319         kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
3320         kvm_run->cr8 = kvm_get_cr8(vcpu);
3321         kvm_run->apic_base = kvm_get_apic_base(vcpu);
3322         if (irqchip_in_kernel(vcpu->kvm))
3323                 kvm_run->ready_for_interrupt_injection = 1;
3324         else
3325                 kvm_run->ready_for_interrupt_injection =
3326                         kvm_arch_interrupt_allowed(vcpu) &&
3327                         !kvm_cpu_has_interrupt(vcpu) &&
3328                         !kvm_event_needs_reinjection(vcpu);
3329 }
3330
3331 static void vapic_enter(struct kvm_vcpu *vcpu)
3332 {
3333         struct kvm_lapic *apic = vcpu->arch.apic;
3334         struct page *page;
3335
3336         if (!apic || !apic->vapic_addr)
3337                 return;
3338
3339         page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
3340
3341         vcpu->arch.apic->vapic_page = page;
3342 }
3343
3344 static void vapic_exit(struct kvm_vcpu *vcpu)
3345 {
3346         struct kvm_lapic *apic = vcpu->arch.apic;
3347
3348         if (!apic || !apic->vapic_addr)
3349                 return;
3350
3351         down_read(&vcpu->kvm->slots_lock);
3352         kvm_release_page_dirty(apic->vapic_page);
3353         mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
3354         up_read(&vcpu->kvm->slots_lock);
3355 }
3356
3357 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
3358 {
3359         int max_irr, tpr;
3360
3361         if (!kvm_x86_ops->update_cr8_intercept)
3362                 return;
3363
3364         if (!vcpu->arch.apic->vapic_addr)
3365                 max_irr = kvm_lapic_find_highest_irr(vcpu);
3366         else
3367                 max_irr = -1;
3368
3369         if (max_irr != -1)
3370                 max_irr >>= 4;
3371
3372         tpr = kvm_lapic_get_cr8(vcpu);
3373
3374         kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
3375 }
3376
3377 static void inject_pending_irq(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3378 {
3379         /* try to reinject previous events if any */
3380         if (vcpu->arch.nmi_injected) {
3381                 kvm_x86_ops->set_nmi(vcpu);
3382                 return;
3383         }
3384
3385         if (vcpu->arch.interrupt.pending) {
3386                 kvm_x86_ops->set_irq(vcpu);
3387                 return;
3388         }
3389
3390         /* try to inject new event if pending */
3391         if (vcpu->arch.nmi_pending) {
3392                 if (kvm_x86_ops->nmi_allowed(vcpu)) {
3393                         vcpu->arch.nmi_pending = false;
3394                         vcpu->arch.nmi_injected = true;
3395                         kvm_x86_ops->set_nmi(vcpu);
3396                 }
3397         } else if (kvm_cpu_has_interrupt(vcpu)) {
3398                 if (kvm_x86_ops->interrupt_allowed(vcpu)) {
3399                         kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
3400                                             false);
3401                         kvm_x86_ops->set_irq(vcpu);
3402                 }
3403         }
3404 }
3405
3406 static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3407 {
3408         int r;
3409         bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
3410                 kvm_run->request_interrupt_window;
3411
3412         if (vcpu->requests)
3413                 if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
3414                         kvm_mmu_unload(vcpu);
3415
3416         r = kvm_mmu_reload(vcpu);
3417         if (unlikely(r))
3418                 goto out;
3419
3420         if (vcpu->requests) {
3421                 if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
3422                         __kvm_migrate_timers(vcpu);
3423                 if (test_and_clear_bit(KVM_REQ_KVMCLOCK_UPDATE, &vcpu->requests))
3424                         kvm_write_guest_time(vcpu);
3425                 if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests))
3426                         kvm_mmu_sync_roots(vcpu);
3427                 if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
3428                         kvm_x86_ops->tlb_flush(vcpu);
3429                 if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
3430                                        &vcpu->requests)) {
3431                         kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS;
3432                         r = 0;
3433                         goto out;
3434                 }
3435                 if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) {
3436                         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
3437                         r = 0;
3438                         goto out;
3439                 }
3440         }
3441
3442         preempt_disable();
3443
3444         kvm_x86_ops->prepare_guest_switch(vcpu);
3445         kvm_load_guest_fpu(vcpu);
3446
3447         local_irq_disable();
3448
3449         clear_bit(KVM_REQ_KICK, &vcpu->requests);
3450         smp_mb__after_clear_bit();
3451
3452         if (vcpu->requests || need_resched() || signal_pending(current)) {
3453                 local_irq_enable();
3454                 preempt_enable();
3455                 r = 1;
3456                 goto out;
3457         }
3458
3459         if (vcpu->arch.exception.pending)
3460                 __queue_exception(vcpu);
3461         else
3462                 inject_pending_irq(vcpu, kvm_run);
3463
3464         /* enable NMI/IRQ window open exits if needed */
3465         if (vcpu->arch.nmi_pending)
3466                 kvm_x86_ops->enable_nmi_window(vcpu);
3467         else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
3468                 kvm_x86_ops->enable_irq_window(vcpu);
3469
3470         if (kvm_lapic_enabled(vcpu)) {
3471                 update_cr8_intercept(vcpu);
3472                 kvm_lapic_sync_to_vapic(vcpu);
3473         }
3474
3475         up_read(&vcpu->kvm->slots_lock);
3476
3477         kvm_guest_enter();
3478
3479         get_debugreg(vcpu->arch.host_dr6, 6);
3480         get_debugreg(vcpu->arch.host_dr7, 7);
3481         if (unlikely(vcpu->arch.switch_db_regs)) {
3482                 get_debugreg(vcpu->arch.host_db[0], 0);
3483                 get_debugreg(vcpu->arch.host_db[1], 1);
3484                 get_debugreg(vcpu->arch.host_db[2], 2);
3485                 get_debugreg(vcpu->arch.host_db[3], 3);
3486
3487                 set_debugreg(0, 7);
3488                 set_debugreg(vcpu->arch.eff_db[0], 0);
3489                 set_debugreg(vcpu->arch.eff_db[1], 1);
3490                 set_debugreg(vcpu->arch.eff_db[2], 2);
3491                 set_debugreg(vcpu->arch.eff_db[3], 3);
3492         }
3493
3494         KVMTRACE_0D(VMENTRY, vcpu, entryexit);
3495         kvm_x86_ops->run(vcpu, kvm_run);
3496
3497         if (unlikely(vcpu->arch.switch_db_regs)) {
3498                 set_debugreg(0, 7);
3499                 set_debugreg(vcpu->arch.host_db[0], 0);
3500                 set_debugreg(vcpu->arch.host_db[1], 1);
3501                 set_debugreg(vcpu->arch.host_db[2], 2);
3502                 set_debugreg(vcpu->arch.host_db[3], 3);
3503         }
3504         set_debugreg(vcpu->arch.host_dr6, 6);
3505         set_debugreg(vcpu->arch.host_dr7, 7);
3506
3507         set_bit(KVM_REQ_KICK, &vcpu->requests);
3508         local_irq_enable();
3509
3510         ++vcpu->stat.exits;
3511
3512         /*
3513          * We must have an instruction between local_irq_enable() and
3514          * kvm_guest_exit(), so the timer interrupt isn't delayed by
3515          * the interrupt shadow.  The stat.exits increment will do nicely.
3516          * But we need to prevent reordering, hence this barrier():
3517          */
3518         barrier();
3519
3520         kvm_guest_exit();
3521
3522         preempt_enable();
3523
3524         down_read(&vcpu->kvm->slots_lock);
3525
3526         /*
3527          * Profile KVM exit RIPs:
3528          */
3529         if (unlikely(prof_on == KVM_PROFILING)) {
3530                 unsigned long rip = kvm_rip_read(vcpu);
3531                 profile_hit(KVM_PROFILING, (void *)rip);
3532         }
3533
3534
3535         kvm_lapic_sync_from_vapic(vcpu);
3536
3537         r = kvm_x86_ops->handle_exit(kvm_run, vcpu);
3538 out:
3539         return r;
3540 }
3541
3542
3543 static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3544 {
3545         int r;
3546
3547         if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
3548                 pr_debug("vcpu %d received sipi with vector # %x\n",
3549                          vcpu->vcpu_id, vcpu->arch.sipi_vector);
3550                 kvm_lapic_reset(vcpu);
3551                 r = kvm_arch_vcpu_reset(vcpu);
3552                 if (r)
3553                         return r;
3554                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
3555         }
3556
3557         down_read(&vcpu->kvm->slots_lock);
3558         vapic_enter(vcpu);
3559
3560         r = 1;
3561         while (r > 0) {
3562                 if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
3563                         r = vcpu_enter_guest(vcpu, kvm_run);
3564                 else {
3565                         up_read(&vcpu->kvm->slots_lock);
3566                         kvm_vcpu_block(vcpu);
3567                         down_read(&vcpu->kvm->slots_lock);
3568                         if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests))
3569                         {
3570                                 switch(vcpu->arch.mp_state) {
3571                                 case KVM_MP_STATE_HALTED:
3572                                         vcpu->arch.mp_state =
3573                                                 KVM_MP_STATE_RUNNABLE;
3574                                 case KVM_MP_STATE_RUNNABLE:
3575                                         break;
3576                                 case KVM_MP_STATE_SIPI_RECEIVED:
3577                                 default:
3578                                         r = -EINTR;
3579                                         break;
3580                                 }
3581                         }
3582                 }
3583
3584                 if (r <= 0)
3585                         break;
3586
3587                 clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
3588                 if (kvm_cpu_has_pending_timer(vcpu))
3589                         kvm_inject_pending_timer_irqs(vcpu);
3590
3591                 if (dm_request_for_irq_injection(vcpu, kvm_run)) {
3592                         r = -EINTR;
3593                         kvm_run->exit_reason = KVM_EXIT_INTR;
3594                         ++vcpu->stat.request_irq_exits;
3595                 }
3596                 if (signal_pending(current)) {
3597                         r = -EINTR;
3598                         kvm_run->exit_reason = KVM_EXIT_INTR;
3599                         ++vcpu->stat.signal_exits;
3600                 }
3601                 if (need_resched()) {
3602                         up_read(&vcpu->kvm->slots_lock);
3603                         kvm_resched(vcpu);
3604                         down_read(&vcpu->kvm->slots_lock);
3605                 }
3606         }
3607
3608         up_read(&vcpu->kvm->slots_lock);
3609         post_kvm_run_save(vcpu, kvm_run);
3610
3611         vapic_exit(vcpu);
3612
3613         return r;
3614 }
3615
3616 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3617 {
3618         int r;
3619         sigset_t sigsaved;
3620
3621         vcpu_load(vcpu);
3622
3623         if (vcpu->sigset_active)
3624                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
3625
3626         if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
3627                 kvm_vcpu_block(vcpu);
3628                 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
3629                 r = -EAGAIN;
3630                 goto out;
3631         }
3632
3633         /* re-sync apic's tpr */
3634         if (!irqchip_in_kernel(vcpu->kvm))
3635                 kvm_set_cr8(vcpu, kvm_run->cr8);
3636
3637         if (vcpu->arch.pio.cur_count) {
3638                 r = complete_pio(vcpu);
3639                 if (r)
3640                         goto out;
3641         }
3642 #if CONFIG_HAS_IOMEM
3643         if (vcpu->mmio_needed) {
3644                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
3645                 vcpu->mmio_read_completed = 1;
3646                 vcpu->mmio_needed = 0;
3647
3648                 down_read(&vcpu->kvm->slots_lock);
3649                 r = emulate_instruction(vcpu, kvm_run,
3650                                         vcpu->arch.mmio_fault_cr2, 0,
3651                                         EMULTYPE_NO_DECODE);
3652                 up_read(&vcpu->kvm->slots_lock);
3653                 if (r == EMULATE_DO_MMIO) {
3654                         /*
3655                          * Read-modify-write.  Back to userspace.
3656                          */
3657                         r = 0;
3658                         goto out;
3659                 }
3660         }
3661 #endif
3662         if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
3663                 kvm_register_write(vcpu, VCPU_REGS_RAX,
3664                                      kvm_run->hypercall.ret);
3665
3666         r = __vcpu_run(vcpu, kvm_run);
3667
3668 out:
3669         if (vcpu->sigset_active)
3670                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
3671
3672         vcpu_put(vcpu);
3673         return r;
3674 }
3675
3676 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3677 {
3678         vcpu_load(vcpu);
3679
3680         regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
3681         regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
3682         regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
3683         regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
3684         regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
3685         regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
3686         regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
3687         regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
3688 #ifdef CONFIG_X86_64
3689         regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
3690         regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
3691         regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
3692         regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
3693         regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
3694         regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
3695         regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
3696         regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
3697 #endif
3698
3699         regs->rip = kvm_rip_read(vcpu);
3700         regs->rflags = kvm_x86_ops->get_rflags(vcpu);
3701
3702         /*
3703          * Don't leak debug flags in case they were set for guest debugging
3704          */
3705         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
3706                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
3707
3708         vcpu_put(vcpu);
3709
3710         return 0;
3711 }
3712
3713 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3714 {
3715         vcpu_load(vcpu);
3716
3717         kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
3718         kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
3719         kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
3720         kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
3721         kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
3722         kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
3723         kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
3724         kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
3725 #ifdef CONFIG_X86_64
3726         kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
3727         kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
3728         kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
3729         kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
3730         kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
3731         kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
3732         kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
3733         kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
3734
3735 #endif
3736
3737         kvm_rip_write(vcpu, regs->rip);
3738         kvm_x86_ops->set_rflags(vcpu, regs->rflags);
3739
3740
3741         vcpu->arch.exception.pending = false;
3742
3743         vcpu_put(vcpu);
3744
3745         return 0;
3746 }
3747
3748 void kvm_get_segment(struct kvm_vcpu *vcpu,
3749                      struct kvm_segment *var, int seg)
3750 {
3751         kvm_x86_ops->get_segment(vcpu, var, seg);
3752 }
3753
3754 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
3755 {
3756         struct kvm_segment cs;
3757
3758         kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
3759         *db = cs.db;
3760         *l = cs.l;
3761 }
3762 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
3763
3764 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3765                                   struct kvm_sregs *sregs)
3766 {
3767         struct descriptor_table dt;
3768
3769         vcpu_load(vcpu);
3770
3771         kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
3772         kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
3773         kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
3774         kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
3775         kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
3776         kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
3777
3778         kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
3779         kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
3780
3781         kvm_x86_ops->get_idt(vcpu, &dt);
3782         sregs->idt.limit = dt.limit;
3783         sregs->idt.base = dt.base;
3784         kvm_x86_ops->get_gdt(vcpu, &dt);
3785         sregs->gdt.limit = dt.limit;
3786         sregs->gdt.base = dt.base;
3787
3788         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
3789         sregs->cr0 = vcpu->arch.cr0;
3790         sregs->cr2 = vcpu->arch.cr2;
3791         sregs->cr3 = vcpu->arch.cr3;
3792         sregs->cr4 = vcpu->arch.cr4;
3793         sregs->cr8 = kvm_get_cr8(vcpu);
3794         sregs->efer = vcpu->arch.shadow_efer;
3795         sregs->apic_base = kvm_get_apic_base(vcpu);
3796
3797         memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
3798
3799         if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
3800                 set_bit(vcpu->arch.interrupt.nr,
3801                         (unsigned long *)sregs->interrupt_bitmap);
3802
3803         vcpu_put(vcpu);
3804
3805         return 0;
3806 }
3807
3808 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3809                                     struct kvm_mp_state *mp_state)
3810 {
3811         vcpu_load(vcpu);
3812         mp_state->mp_state = vcpu->arch.mp_state;
3813         vcpu_put(vcpu);
3814         return 0;
3815 }
3816
3817 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3818                                     struct kvm_mp_state *mp_state)
3819 {
3820         vcpu_load(vcpu);
3821         vcpu->arch.mp_state = mp_state->mp_state;
3822         vcpu_put(vcpu);
3823         return 0;
3824 }
3825
3826 static void kvm_set_segment(struct kvm_vcpu *vcpu,
3827                         struct kvm_segment *var, int seg)
3828 {
3829         kvm_x86_ops->set_segment(vcpu, var, seg);
3830 }
3831
3832 static void seg_desct_to_kvm_desct(struct desc_struct *seg_desc, u16 selector,
3833                                    struct kvm_segment *kvm_desct)
3834 {
3835         kvm_desct->base = seg_desc->base0;
3836         kvm_desct->base |= seg_desc->base1 << 16;
3837         kvm_desct->base |= seg_desc->base2 << 24;
3838         kvm_desct->limit = seg_desc->limit0;
3839         kvm_desct->limit |= seg_desc->limit << 16;
3840         if (seg_desc->g) {
3841                 kvm_desct->limit <<= 12;
3842                 kvm_desct->limit |= 0xfff;
3843         }
3844         kvm_desct->selector = selector;
3845         kvm_desct->type = seg_desc->type;
3846         kvm_desct->present = seg_desc->p;
3847         kvm_desct->dpl = seg_desc->dpl;
3848         kvm_desct->db = seg_desc->d;
3849         kvm_desct->s = seg_desc->s;
3850         kvm_desct->l = seg_desc->l;
3851         kvm_desct->g = seg_desc->g;
3852         kvm_desct->avl = seg_desc->avl;
3853         if (!selector)
3854                 kvm_desct->unusable = 1;
3855         else
3856                 kvm_desct->unusable = 0;
3857         kvm_desct->padding = 0;
3858 }
3859
3860 static void get_segment_descriptor_dtable(struct kvm_vcpu *vcpu,
3861                                           u16 selector,
3862                                           struct descriptor_table *dtable)
3863 {
3864         if (selector & 1 << 2) {
3865                 struct kvm_segment kvm_seg;
3866
3867                 kvm_get_segment(vcpu, &kvm_seg, VCPU_SREG_LDTR);
3868
3869                 if (kvm_seg.unusable)
3870                         dtable->limit = 0;
3871                 else
3872                         dtable->limit = kvm_seg.limit;
3873                 dtable->base = kvm_seg.base;
3874         }
3875         else
3876                 kvm_x86_ops->get_gdt(vcpu, dtable);
3877 }
3878
3879 /* allowed just for 8 bytes segments */
3880 static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
3881                                          struct desc_struct *seg_desc)
3882 {
3883         gpa_t gpa;
3884         struct descriptor_table dtable;
3885         u16 index = selector >> 3;
3886
3887         get_segment_descriptor_dtable(vcpu, selector, &dtable);
3888
3889         if (dtable.limit < index * 8 + 7) {
3890                 kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc);
3891                 return 1;
3892         }
3893         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base);
3894         gpa += index * 8;
3895         return kvm_read_guest(vcpu->kvm, gpa, seg_desc, 8);
3896 }
3897
3898 /* allowed just for 8 bytes segments */
3899 static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
3900                                          struct desc_struct *seg_desc)
3901 {
3902         gpa_t gpa;
3903         struct descriptor_table dtable;
3904         u16 index = selector >> 3;
3905
3906         get_segment_descriptor_dtable(vcpu, selector, &dtable);
3907
3908         if (dtable.limit < index * 8 + 7)
3909                 return 1;
3910         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base);
3911         gpa += index * 8;
3912         return kvm_write_guest(vcpu->kvm, gpa, seg_desc, 8);
3913 }
3914
3915 static u32 get_tss_base_addr(struct kvm_vcpu *vcpu,
3916                              struct desc_struct *seg_desc)
3917 {
3918         u32 base_addr;
3919
3920         base_addr = seg_desc->base0;
3921         base_addr |= (seg_desc->base1 << 16);
3922         base_addr |= (seg_desc->base2 << 24);
3923
3924         return vcpu->arch.mmu.gva_to_gpa(vcpu, base_addr);
3925 }
3926
3927 static u16 get_segment_selector(struct kvm_vcpu *vcpu, int seg)
3928 {
3929         struct kvm_segment kvm_seg;
3930
3931         kvm_get_segment(vcpu, &kvm_seg, seg);
3932         return kvm_seg.selector;
3933 }
3934
3935 static int load_segment_descriptor_to_kvm_desct(struct kvm_vcpu *vcpu,
3936                                                 u16 selector,
3937                                                 struct kvm_segment *kvm_seg)
3938 {
3939         struct desc_struct seg_desc;
3940
3941         if (load_guest_segment_descriptor(vcpu, selector, &seg_desc))
3942                 return 1;
3943         seg_desct_to_kvm_desct(&seg_desc, selector, kvm_seg);
3944         return 0;
3945 }
3946
3947 static int kvm_load_realmode_segment(struct kvm_vcpu *vcpu, u16 selector, int seg)
3948 {
3949         struct kvm_segment segvar = {
3950                 .base = selector << 4,
3951                 .limit = 0xffff,
3952                 .selector = selector,
3953                 .type = 3,
3954                 .present = 1,
3955                 .dpl = 3,
3956                 .db = 0,
3957                 .s = 1,
3958                 .l = 0,
3959                 .g = 0,
3960                 .avl = 0,
3961                 .unusable = 0,
3962         };
3963         kvm_x86_ops->set_segment(vcpu, &segvar, seg);
3964         return 0;
3965 }
3966
3967 int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
3968                                 int type_bits, int seg)
3969 {
3970         struct kvm_segment kvm_seg;
3971
3972         if (!(vcpu->arch.cr0 & X86_CR0_PE))
3973                 return kvm_load_realmode_segment(vcpu, selector, seg);
3974         if (load_segment_descriptor_to_kvm_desct(vcpu, selector, &kvm_seg))
3975                 return 1;
3976         kvm_seg.type |= type_bits;
3977
3978         if (seg != VCPU_SREG_SS && seg != VCPU_SREG_CS &&
3979             seg != VCPU_SREG_LDTR)
3980                 if (!kvm_seg.s)
3981                         kvm_seg.unusable = 1;
3982
3983         kvm_set_segment(vcpu, &kvm_seg, seg);
3984         return 0;
3985 }
3986
3987 static void save_state_to_tss32(struct kvm_vcpu *vcpu,
3988                                 struct tss_segment_32 *tss)
3989 {
3990         tss->cr3 = vcpu->arch.cr3;
3991         tss->eip = kvm_rip_read(vcpu);
3992         tss->eflags = kvm_x86_ops->get_rflags(vcpu);
3993         tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
3994         tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
3995         tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX);
3996         tss->ebx = kvm_register_read(vcpu, VCPU_REGS_RBX);
3997         tss->esp = kvm_register_read(vcpu, VCPU_REGS_RSP);
3998         tss->ebp = kvm_register_read(vcpu, VCPU_REGS_RBP);
3999         tss->esi = kvm_register_read(vcpu, VCPU_REGS_RSI);
4000         tss->edi = kvm_register_read(vcpu, VCPU_REGS_RDI);
4001         tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
4002         tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
4003         tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
4004         tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
4005         tss->fs = get_segment_selector(vcpu, VCPU_SREG_FS);
4006         tss->gs = get_segment_selector(vcpu, VCPU_SREG_GS);
4007         tss->ldt_selector = get_segment_selector(vcpu, VCPU_SREG_LDTR);
4008 }
4009
4010 static int load_state_from_tss32(struct kvm_vcpu *vcpu,
4011                                   struct tss_segment_32 *tss)
4012 {
4013         kvm_set_cr3(vcpu, tss->cr3);
4014
4015         kvm_rip_write(vcpu, tss->eip);
4016         kvm_x86_ops->set_rflags(vcpu, tss->eflags | 2);
4017
4018         kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax);
4019         kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx);
4020         kvm_register_write(vcpu, VCPU_REGS_RDX, tss->edx);
4021         kvm_register_write(vcpu, VCPU_REGS_RBX, tss->ebx);
4022         kvm_register_write(vcpu, VCPU_REGS_RSP, tss->esp);
4023         kvm_register_write(vcpu, VCPU_REGS_RBP, tss->ebp);
4024         kvm_register_write(vcpu, VCPU_REGS_RSI, tss->esi);
4025         kvm_register_write(vcpu, VCPU_REGS_RDI, tss->edi);
4026
4027         if (kvm_load_segment_descriptor(vcpu, tss->ldt_selector, 0, VCPU_SREG_LDTR))
4028                 return 1;
4029
4030         if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
4031                 return 1;
4032
4033         if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
4034                 return 1;
4035
4036         if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
4037                 return 1;
4038
4039         if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
4040                 return 1;
4041
4042         if (kvm_load_segment_descriptor(vcpu, tss->fs, 1, VCPU_SREG_FS))
4043                 return 1;
4044
4045         if (kvm_load_segment_descriptor(vcpu, tss->gs, 1, VCPU_SREG_GS))
4046                 return 1;
4047         return 0;
4048 }
4049
4050 static void save_state_to_tss16(struct kvm_vcpu *vcpu,
4051                                 struct tss_segment_16 *tss)
4052 {
4053         tss->ip = kvm_rip_read(vcpu);
4054         tss->flag = kvm_x86_ops->get_rflags(vcpu);
4055         tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX);
4056         tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX);
4057         tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX);
4058         tss->bx = kvm_register_read(vcpu, VCPU_REGS_RBX);
4059         tss->sp = kvm_register_read(vcpu, VCPU_REGS_RSP);
4060         tss->bp = kvm_register_read(vcpu, VCPU_REGS_RBP);
4061         tss->si = kvm_register_read(vcpu, VCPU_REGS_RSI);
4062         tss->di = kvm_register_read(vcpu, VCPU_REGS_RDI);
4063
4064         tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
4065         tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
4066         tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
4067         tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
4068         tss->ldt = get_segment_selector(vcpu, VCPU_SREG_LDTR);
4069         tss->prev_task_link = get_segment_selector(vcpu, VCPU_SREG_TR);
4070 }
4071
4072 static int load_state_from_tss16(struct kvm_vcpu *vcpu,
4073                                  struct tss_segment_16 *tss)
4074 {
4075         kvm_rip_write(vcpu, tss->ip);
4076         kvm_x86_ops->set_rflags(vcpu, tss->flag | 2);
4077         kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax);
4078         kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx);
4079         kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx);
4080         kvm_register_write(vcpu, VCPU_REGS_RBX, tss->bx);
4081         kvm_register_write(vcpu, VCPU_REGS_RSP, tss->sp);
4082         kvm_register_write(vcpu, VCPU_REGS_RBP, tss->bp);
4083         kvm_register_write(vcpu, VCPU_REGS_RSI, tss->si);
4084         kvm_register_write(vcpu, VCPU_REGS_RDI, tss->di);
4085
4086         if (kvm_load_segment_descriptor(vcpu, tss->ldt, 0, VCPU_SREG_LDTR))
4087                 return 1;
4088
4089         if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
4090                 return 1;
4091
4092         if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
4093                 return 1;
4094
4095         if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
4096                 return 1;
4097
4098         if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
4099                 return 1;
4100         return 0;
4101 }
4102
4103 static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector,
4104                               u16 old_tss_sel, u32 old_tss_base,
4105                               struct desc_struct *nseg_desc)
4106 {
4107         struct tss_segment_16 tss_segment_16;
4108         int ret = 0;
4109
4110         if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
4111                            sizeof tss_segment_16))
4112                 goto out;
4113
4114         save_state_to_tss16(vcpu, &tss_segment_16);
4115
4116         if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
4117                             sizeof tss_segment_16))
4118                 goto out;
4119
4120         if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
4121                            &tss_segment_16, sizeof tss_segment_16))
4122                 goto out;
4123
4124         if (old_tss_sel != 0xffff) {
4125                 tss_segment_16.prev_task_link = old_tss_sel;
4126
4127                 if (kvm_write_guest(vcpu->kvm,
4128                                     get_tss_base_addr(vcpu, nseg_desc),
4129                                     &tss_segment_16.prev_task_link,
4130                                     sizeof tss_segment_16.prev_task_link))
4131                         goto out;
4132         }
4133
4134         if (load_state_from_tss16(vcpu, &tss_segment_16))
4135                 goto out;
4136
4137         ret = 1;
4138 out:
4139         return ret;
4140 }
4141
4142 static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector,
4143                        u16 old_tss_sel, u32 old_tss_base,
4144                        struct desc_struct *nseg_desc)
4145 {
4146         struct tss_segment_32 tss_segment_32;
4147         int ret = 0;
4148
4149         if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
4150                            sizeof tss_segment_32))
4151                 goto out;
4152
4153         save_state_to_tss32(vcpu, &tss_segment_32);
4154
4155         if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
4156                             sizeof tss_segment_32))
4157                 goto out;
4158
4159         if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
4160                            &tss_segment_32, sizeof tss_segment_32))
4161                 goto out;
4162
4163         if (old_tss_sel != 0xffff) {
4164                 tss_segment_32.prev_task_link = old_tss_sel;
4165
4166                 if (kvm_write_guest(vcpu->kvm,
4167                                     get_tss_base_addr(vcpu, nseg_desc),
4168                                     &tss_segment_32.prev_task_link,
4169                                     sizeof tss_segment_32.prev_task_link))
4170                         goto out;
4171         }
4172
4173         if (load_state_from_tss32(vcpu, &tss_segment_32))
4174                 goto out;
4175
4176         ret = 1;
4177 out:
4178         return ret;
4179 }
4180
4181 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
4182 {
4183         struct kvm_segment tr_seg;
4184         struct desc_struct cseg_desc;
4185         struct desc_struct nseg_desc;
4186         int ret = 0;
4187         u32 old_tss_base = get_segment_base(vcpu, VCPU_SREG_TR);
4188         u16 old_tss_sel = get_segment_selector(vcpu, VCPU_SREG_TR);
4189
4190         old_tss_base = vcpu->arch.mmu.gva_to_gpa(vcpu, old_tss_base);
4191
4192         /* FIXME: Handle errors. Failure to read either TSS or their
4193          * descriptors should generate a pagefault.
4194          */
4195         if (load_guest_segment_descriptor(vcpu, tss_selector, &nseg_desc))
4196                 goto out;
4197
4198         if (load_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc))
4199                 goto out;
4200
4201         if (reason != TASK_SWITCH_IRET) {
4202                 int cpl;
4203
4204                 cpl = kvm_x86_ops->get_cpl(vcpu);
4205                 if ((tss_selector & 3) > nseg_desc.dpl || cpl > nseg_desc.dpl) {
4206                         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
4207                         return 1;
4208                 }
4209         }
4210
4211         if (!nseg_desc.p || (nseg_desc.limit0 | nseg_desc.limit << 16) < 0x67) {
4212                 kvm_queue_exception_e(vcpu, TS_VECTOR, tss_selector & 0xfffc);
4213                 return 1;
4214         }
4215
4216         if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
4217                 cseg_desc.type &= ~(1 << 1); //clear the B flag
4218                 save_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc);
4219         }
4220
4221         if (reason == TASK_SWITCH_IRET) {
4222                 u32 eflags = kvm_x86_ops->get_rflags(vcpu);
4223                 kvm_x86_ops->set_rflags(vcpu, eflags & ~X86_EFLAGS_NT);
4224         }
4225
4226         /* set back link to prev task only if NT bit is set in eflags
4227            note that old_tss_sel is not used afetr this point */
4228         if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
4229                 old_tss_sel = 0xffff;
4230
4231         /* set back link to prev task only if NT bit is set in eflags
4232            note that old_tss_sel is not used afetr this point */
4233         if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
4234                 old_tss_sel = 0xffff;
4235
4236         if (nseg_desc.type & 8)
4237                 ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_sel,
4238                                          old_tss_base, &nseg_desc);
4239         else
4240                 ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_sel,
4241                                          old_tss_base, &nseg_desc);
4242
4243         if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) {
4244                 u32 eflags = kvm_x86_ops->get_rflags(vcpu);
4245                 kvm_x86_ops->set_rflags(vcpu, eflags | X86_EFLAGS_NT);
4246         }
4247
4248         if (reason != TASK_SWITCH_IRET) {
4249                 nseg_desc.type |= (1 << 1);
4250                 save_guest_segment_descriptor(vcpu, tss_selector,
4251                                               &nseg_desc);
4252         }
4253
4254         kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 | X86_CR0_TS);
4255         seg_desct_to_kvm_desct(&nseg_desc, tss_selector, &tr_seg);
4256         tr_seg.type = 11;
4257         kvm_set_segment(vcpu, &tr_seg, VCPU_SREG_TR);
4258 out:
4259         return ret;
4260 }
4261 EXPORT_SYMBOL_GPL(kvm_task_switch);
4262
4263 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
4264                                   struct kvm_sregs *sregs)
4265 {
4266         int mmu_reset_needed = 0;
4267         int pending_vec, max_bits;
4268         struct descriptor_table dt;
4269
4270         vcpu_load(vcpu);
4271
4272         dt.limit = sregs->idt.limit;
4273         dt.base = sregs->idt.base;
4274         kvm_x86_ops->set_idt(vcpu, &dt);
4275         dt.limit = sregs->gdt.limit;
4276         dt.base = sregs->gdt.base;
4277         kvm_x86_ops->set_gdt(vcpu, &dt);
4278
4279         vcpu->arch.cr2 = sregs->cr2;
4280         mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
4281
4282         down_read(&vcpu->kvm->slots_lock);
4283         if (gfn_to_memslot(vcpu->kvm, sregs->cr3 >> PAGE_SHIFT))
4284                 vcpu->arch.cr3 = sregs->cr3;
4285         else
4286                 set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
4287         up_read(&vcpu->kvm->slots_lock);
4288
4289         kvm_set_cr8(vcpu, sregs->cr8);
4290
4291         mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer;
4292         kvm_x86_ops->set_efer(vcpu, sregs->efer);
4293         kvm_set_apic_base(vcpu, sregs->apic_base);
4294
4295         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
4296
4297         mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0;
4298         kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
4299         vcpu->arch.cr0 = sregs->cr0;
4300
4301         mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4;
4302         kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
4303         if (!is_long_mode(vcpu) && is_pae(vcpu))
4304                 load_pdptrs(vcpu, vcpu->arch.cr3);
4305
4306         if (mmu_reset_needed)
4307                 kvm_mmu_reset_context(vcpu);
4308
4309         max_bits = (sizeof sregs->interrupt_bitmap) << 3;
4310         pending_vec = find_first_bit(
4311                 (const unsigned long *)sregs->interrupt_bitmap, max_bits);
4312         if (pending_vec < max_bits) {
4313                 kvm_queue_interrupt(vcpu, pending_vec, false);
4314                 pr_debug("Set back pending irq %d\n", pending_vec);
4315                 if (irqchip_in_kernel(vcpu->kvm))
4316                         kvm_pic_clear_isr_ack(vcpu->kvm);
4317         }
4318
4319         kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
4320         kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
4321         kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
4322         kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
4323         kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
4324         kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
4325
4326         kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
4327         kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
4328
4329         /* Older userspace won't unhalt the vcpu on reset. */
4330         if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
4331             sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
4332             !(vcpu->arch.cr0 & X86_CR0_PE))
4333                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
4334
4335         vcpu_put(vcpu);
4336
4337         return 0;
4338 }
4339
4340 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
4341                                         struct kvm_guest_debug *dbg)
4342 {
4343         int i, r;
4344
4345         vcpu_load(vcpu);
4346
4347         if ((dbg->control & (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP)) ==
4348             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP)) {
4349                 for (i = 0; i < KVM_NR_DB_REGS; ++i)
4350                         vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
4351                 vcpu->arch.switch_db_regs =
4352                         (dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
4353         } else {
4354                 for (i = 0; i < KVM_NR_DB_REGS; i++)
4355                         vcpu->arch.eff_db[i] = vcpu->arch.db[i];
4356                 vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
4357         }
4358
4359         r = kvm_x86_ops->set_guest_debug(vcpu, dbg);
4360
4361         if (dbg->control & KVM_GUESTDBG_INJECT_DB)
4362                 kvm_queue_exception(vcpu, DB_VECTOR);
4363         else if (dbg->control & KVM_GUESTDBG_INJECT_BP)
4364                 kvm_queue_exception(vcpu, BP_VECTOR);
4365
4366         vcpu_put(vcpu);
4367
4368         return r;
4369 }
4370
4371 /*
4372  * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
4373  * we have asm/x86/processor.h
4374  */
4375 struct fxsave {
4376         u16     cwd;
4377         u16     swd;
4378         u16     twd;
4379         u16     fop;
4380         u64     rip;
4381         u64     rdp;
4382         u32     mxcsr;
4383         u32     mxcsr_mask;
4384         u32     st_space[32];   /* 8*16 bytes for each FP-reg = 128 bytes */
4385 #ifdef CONFIG_X86_64
4386         u32     xmm_space[64];  /* 16*16 bytes for each XMM-reg = 256 bytes */
4387 #else
4388         u32     xmm_space[32];  /* 8*16 bytes for each XMM-reg = 128 bytes */
4389 #endif
4390 };
4391
4392 /*
4393  * Translate a guest virtual address to a guest physical address.
4394  */
4395 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
4396                                     struct kvm_translation *tr)
4397 {
4398         unsigned long vaddr = tr->linear_address;
4399         gpa_t gpa;
4400
4401         vcpu_load(vcpu);
4402         down_read(&vcpu->kvm->slots_lock);
4403         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr);
4404         up_read(&vcpu->kvm->slots_lock);
4405         tr->physical_address = gpa;
4406         tr->valid = gpa != UNMAPPED_GVA;
4407         tr->writeable = 1;
4408         tr->usermode = 0;
4409         vcpu_put(vcpu);
4410
4411         return 0;
4412 }
4413
4414 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4415 {
4416         struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
4417
4418         vcpu_load(vcpu);
4419
4420         memcpy(fpu->fpr, fxsave->st_space, 128);
4421         fpu->fcw = fxsave->cwd;
4422         fpu->fsw = fxsave->swd;
4423         fpu->ftwx = fxsave->twd;
4424         fpu->last_opcode = fxsave->fop;
4425         fpu->last_ip = fxsave->rip;
4426         fpu->last_dp = fxsave->rdp;
4427         memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
4428
4429         vcpu_put(vcpu);
4430
4431         return 0;
4432 }
4433
4434 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4435 {
4436         struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
4437
4438         vcpu_load(vcpu);
4439
4440         memcpy(fxsave->st_space, fpu->fpr, 128);
4441         fxsave->cwd = fpu->fcw;
4442         fxsave->swd = fpu->fsw;
4443         fxsave->twd = fpu->ftwx;
4444         fxsave->fop = fpu->last_opcode;
4445         fxsave->rip = fpu->last_ip;
4446         fxsave->rdp = fpu->last_dp;
4447         memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
4448
4449         vcpu_put(vcpu);
4450
4451         return 0;
4452 }
4453
4454 void fx_init(struct kvm_vcpu *vcpu)
4455 {
4456         unsigned after_mxcsr_mask;
4457
4458         /*
4459          * Touch the fpu the first time in non atomic context as if
4460          * this is the first fpu instruction the exception handler
4461          * will fire before the instruction returns and it'll have to
4462          * allocate ram with GFP_KERNEL.
4463          */
4464         if (!used_math())
4465                 kvm_fx_save(&vcpu->arch.host_fx_image);
4466
4467         /* Initialize guest FPU by resetting ours and saving into guest's */
4468         preempt_disable();
4469         kvm_fx_save(&vcpu->arch.host_fx_image);
4470         kvm_fx_finit();
4471         kvm_fx_save(&vcpu->arch.guest_fx_image);
4472         kvm_fx_restore(&vcpu->arch.host_fx_image);
4473         preempt_enable();
4474
4475         vcpu->arch.cr0 |= X86_CR0_ET;
4476         after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
4477         vcpu->arch.guest_fx_image.mxcsr = 0x1f80;
4478         memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask,
4479                0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
4480 }
4481 EXPORT_SYMBOL_GPL(fx_init);
4482
4483 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
4484 {
4485         if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
4486                 return;
4487
4488         vcpu->guest_fpu_loaded = 1;
4489         kvm_fx_save(&vcpu->arch.host_fx_image);
4490         kvm_fx_restore(&vcpu->arch.guest_fx_image);
4491 }
4492 EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
4493
4494 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
4495 {
4496         if (!vcpu->guest_fpu_loaded)
4497                 return;
4498
4499         vcpu->guest_fpu_loaded = 0;
4500         kvm_fx_save(&vcpu->arch.guest_fx_image);
4501         kvm_fx_restore(&vcpu->arch.host_fx_image);
4502         ++vcpu->stat.fpu_reload;
4503 }
4504 EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
4505
4506 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
4507 {
4508         if (vcpu->arch.time_page) {
4509                 kvm_release_page_dirty(vcpu->arch.time_page);
4510                 vcpu->arch.time_page = NULL;
4511         }
4512
4513         kvm_x86_ops->vcpu_free(vcpu);
4514 }
4515
4516 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
4517                                                 unsigned int id)
4518 {
4519         return kvm_x86_ops->vcpu_create(kvm, id);
4520 }
4521
4522 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
4523 {
4524         int r;
4525
4526         /* We do fxsave: this must be aligned. */
4527         BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF);
4528
4529         vcpu->arch.mtrr_state.have_fixed = 1;
4530         vcpu_load(vcpu);
4531         r = kvm_arch_vcpu_reset(vcpu);
4532         if (r == 0)
4533                 r = kvm_mmu_setup(vcpu);
4534         vcpu_put(vcpu);
4535         if (r < 0)
4536                 goto free_vcpu;
4537
4538         return 0;
4539 free_vcpu:
4540         kvm_x86_ops->vcpu_free(vcpu);
4541         return r;
4542 }
4543
4544 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
4545 {
4546         vcpu_load(vcpu);
4547         kvm_mmu_unload(vcpu);
4548         vcpu_put(vcpu);
4549
4550         kvm_x86_ops->vcpu_free(vcpu);
4551 }
4552
4553 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
4554 {
4555         vcpu->arch.nmi_pending = false;
4556         vcpu->arch.nmi_injected = false;
4557
4558         vcpu->arch.switch_db_regs = 0;
4559         memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
4560         vcpu->arch.dr6 = DR6_FIXED_1;
4561         vcpu->arch.dr7 = DR7_FIXED_1;
4562
4563         return kvm_x86_ops->vcpu_reset(vcpu);
4564 }
4565
4566 void kvm_arch_hardware_enable(void *garbage)
4567 {
4568         kvm_x86_ops->hardware_enable(garbage);
4569 }
4570
4571 void kvm_arch_hardware_disable(void *garbage)
4572 {
4573         kvm_x86_ops->hardware_disable(garbage);
4574 }
4575
4576 int kvm_arch_hardware_setup(void)
4577 {
4578         return kvm_x86_ops->hardware_setup();
4579 }
4580
4581 void kvm_arch_hardware_unsetup(void)
4582 {
4583         kvm_x86_ops->hardware_unsetup();
4584 }
4585
4586 void kvm_arch_check_processor_compat(void *rtn)
4587 {
4588         kvm_x86_ops->check_processor_compatibility(rtn);
4589 }
4590
4591 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
4592 {
4593         struct page *page;
4594         struct kvm *kvm;
4595         int r;
4596
4597         BUG_ON(vcpu->kvm == NULL);
4598         kvm = vcpu->kvm;
4599
4600         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
4601         if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
4602                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
4603         else
4604                 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
4605
4606         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
4607         if (!page) {
4608                 r = -ENOMEM;
4609                 goto fail;
4610         }
4611         vcpu->arch.pio_data = page_address(page);
4612
4613         r = kvm_mmu_create(vcpu);
4614         if (r < 0)
4615                 goto fail_free_pio_data;
4616
4617         if (irqchip_in_kernel(kvm)) {
4618                 r = kvm_create_lapic(vcpu);
4619                 if (r < 0)
4620                         goto fail_mmu_destroy;
4621         }
4622
4623         vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
4624                                        GFP_KERNEL);
4625         if (!vcpu->arch.mce_banks) {
4626                 r = -ENOMEM;
4627                 goto fail_mmu_destroy;
4628         }
4629         vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
4630
4631         return 0;
4632
4633 fail_mmu_destroy:
4634         kvm_mmu_destroy(vcpu);
4635 fail_free_pio_data:
4636         free_page((unsigned long)vcpu->arch.pio_data);
4637 fail:
4638         return r;
4639 }
4640
4641 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
4642 {
4643         kvm_free_lapic(vcpu);
4644         down_read(&vcpu->kvm->slots_lock);
4645         kvm_mmu_destroy(vcpu);
4646         up_read(&vcpu->kvm->slots_lock);
4647         free_page((unsigned long)vcpu->arch.pio_data);
4648 }
4649
4650 struct  kvm *kvm_arch_create_vm(void)
4651 {
4652         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
4653
4654         if (!kvm)
4655                 return ERR_PTR(-ENOMEM);
4656
4657         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
4658         INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
4659
4660         /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
4661         set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
4662
4663         rdtscll(kvm->arch.vm_init_tsc);
4664
4665         return kvm;
4666 }
4667
4668 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
4669 {
4670         vcpu_load(vcpu);
4671         kvm_mmu_unload(vcpu);
4672         vcpu_put(vcpu);
4673 }
4674
4675 static void kvm_free_vcpus(struct kvm *kvm)
4676 {
4677         unsigned int i;
4678         struct kvm_vcpu *vcpu;
4679
4680         /*
4681          * Unpin any mmu pages first.
4682          */
4683         kvm_for_each_vcpu(i, vcpu, kvm)
4684                 kvm_unload_vcpu_mmu(vcpu);
4685         kvm_for_each_vcpu(i, vcpu, kvm)
4686                 kvm_arch_vcpu_free(vcpu);
4687
4688         mutex_lock(&kvm->lock);
4689         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
4690                 kvm->vcpus[i] = NULL;
4691
4692         atomic_set(&kvm->online_vcpus, 0);
4693         mutex_unlock(&kvm->lock);
4694 }
4695
4696 void kvm_arch_sync_events(struct kvm *kvm)
4697 {
4698         kvm_free_all_assigned_devices(kvm);
4699 }
4700
4701 void kvm_arch_destroy_vm(struct kvm *kvm)
4702 {
4703         kvm_iommu_unmap_guest(kvm);
4704         kvm_free_pit(kvm);
4705         kfree(kvm->arch.vpic);
4706         kfree(kvm->arch.vioapic);
4707         kvm_free_vcpus(kvm);
4708         kvm_free_physmem(kvm);
4709         if (kvm->arch.apic_access_page)
4710                 put_page(kvm->arch.apic_access_page);
4711         if (kvm->arch.ept_identity_pagetable)
4712                 put_page(kvm->arch.ept_identity_pagetable);
4713         kfree(kvm);
4714 }
4715
4716 int kvm_arch_set_memory_region(struct kvm *kvm,
4717                                 struct kvm_userspace_memory_region *mem,
4718                                 struct kvm_memory_slot old,
4719                                 int user_alloc)
4720 {
4721         int npages = mem->memory_size >> PAGE_SHIFT;
4722         struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot];
4723
4724         /*To keep backward compatibility with older userspace,
4725          *x86 needs to hanlde !user_alloc case.
4726          */
4727         if (!user_alloc) {
4728                 if (npages && !old.rmap) {
4729                         unsigned long userspace_addr;
4730
4731                         down_write(&current->mm->mmap_sem);
4732                         userspace_addr = do_mmap(NULL, 0,
4733                                                  npages * PAGE_SIZE,
4734                                                  PROT_READ | PROT_WRITE,
4735                                                  MAP_PRIVATE | MAP_ANONYMOUS,
4736                                                  0);
4737                         up_write(&current->mm->mmap_sem);
4738
4739                         if (IS_ERR((void *)userspace_addr))
4740                                 return PTR_ERR((void *)userspace_addr);
4741
4742                         /* set userspace_addr atomically for kvm_hva_to_rmapp */
4743                         spin_lock(&kvm->mmu_lock);
4744                         memslot->userspace_addr = userspace_addr;
4745                         spin_unlock(&kvm->mmu_lock);
4746                 } else {
4747                         if (!old.user_alloc && old.rmap) {
4748                                 int ret;
4749
4750                                 down_write(&current->mm->mmap_sem);
4751                                 ret = do_munmap(current->mm, old.userspace_addr,
4752                                                 old.npages * PAGE_SIZE);
4753                                 up_write(&current->mm->mmap_sem);
4754                                 if (ret < 0)
4755                                         printk(KERN_WARNING
4756                                        "kvm_vm_ioctl_set_memory_region: "
4757                                        "failed to munmap memory\n");
4758                         }
4759                 }
4760         }
4761
4762         spin_lock(&kvm->mmu_lock);
4763         if (!kvm->arch.n_requested_mmu_pages) {
4764                 unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
4765                 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
4766         }
4767
4768         kvm_mmu_slot_remove_write_access(kvm, mem->slot);
4769         spin_unlock(&kvm->mmu_lock);
4770         kvm_flush_remote_tlbs(kvm);
4771
4772         return 0;
4773 }
4774
4775 void kvm_arch_flush_shadow(struct kvm *kvm)
4776 {
4777         kvm_mmu_zap_all(kvm);
4778         kvm_reload_remote_mmus(kvm);
4779 }
4780
4781 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
4782 {
4783         return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
4784                || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
4785                || vcpu->arch.nmi_pending;
4786 }
4787
4788 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
4789 {
4790         int me;
4791         int cpu = vcpu->cpu;
4792
4793         if (waitqueue_active(&vcpu->wq)) {
4794                 wake_up_interruptible(&vcpu->wq);
4795                 ++vcpu->stat.halt_wakeup;
4796         }
4797
4798         me = get_cpu();
4799         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
4800                 if (!test_and_set_bit(KVM_REQ_KICK, &vcpu->requests))
4801                         smp_send_reschedule(cpu);
4802         put_cpu();
4803 }
4804
4805 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
4806 {
4807         return kvm_x86_ops->interrupt_allowed(vcpu);
4808 }