SAFE public projects git trees. - safe/jmp/linux-2.6/blob - arch/x86/kvm/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "irq.h"
  19 #include "mmu.h"
  20
  21 #include <linux/kvm_host.h>
  22 #include <linux/module.h>
  23 #include <linux/kernel.h>
  24 #include <linux/mm.h>
  25 #include <linux/highmem.h>
  26 #include <linux/sched.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/ftrace_event.h>
  29 #include "kvm_cache_regs.h"
  30 #include "x86.h"
  31
  32 #include <asm/io.h>
  33 #include <asm/desc.h>
  34 #include <asm/vmx.h>
  35 #include <asm/virtext.h>
  36 #include <asm/mce.h>
  37
  38 #include "trace.h"
  39
  40 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  41
  42 MODULE_AUTHOR("Qumranet");
  43 MODULE_LICENSE("GPL");
  44
  45 static int __read_mostly bypass_guest_pf = 1;
  46 module_param(bypass_guest_pf, bool, S_IRUGO);
  47
  48 static int __read_mostly enable_vpid = 1;
  49 module_param_named(vpid, enable_vpid, bool, 0444);
  50
  51 static int __read_mostly flexpriority_enabled = 1;
  52 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  53
  54 static int __read_mostly enable_ept = 1;
  55 module_param_named(ept, enable_ept, bool, S_IRUGO);
  56
  57 static int __read_mostly enable_unrestricted_guest = 1;
  58 module_param_named(unrestricted_guest,
  59                         enable_unrestricted_guest, bool, S_IRUGO);
  60
  61 static int __read_mostly emulate_invalid_guest_state = 0;
  62 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  63
  64 #define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST                           \
  65         (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
  66 #define KVM_GUEST_CR0_MASK                                              \
  67         (KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
  68 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST                         \
  69         (X86_CR0_WP | X86_CR0_NE | X86_CR0_MP)
  70 #define KVM_VM_CR0_ALWAYS_ON                                            \
  71         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
  72 #define KVM_CR4_GUEST_OWNED_BITS                                      \
  73         (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
  74          | X86_CR4_OSXMMEXCPT)
  75
  76 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
  77 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
  78
  79 /*
  80  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
  81  * ple_gap:    upper bound on the amount of time between two successive
  82  *             executions of PAUSE in a loop. Also indicate if ple enabled.
  83  *             According to test, this time is usually small than 41 cycles.
  84  * ple_window: upper bound on the amount of time a guest is allowed to execute
  85  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
  86  *             less than 2^12 cycles
  87  * Time is measured based on a counter that runs at the same rate as the TSC,
  88  * refer SDM volume 3b section 21.6.13 & 22.1.3.
  89  */
  90 #define KVM_VMX_DEFAULT_PLE_GAP    41
  91 #define KVM_VMX_DEFAULT_PLE_WINDOW 4096
  92 static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
  93 module_param(ple_gap, int, S_IRUGO);
  94
  95 static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
  96 module_param(ple_window, int, S_IRUGO);
  97
  98 struct vmcs {
  99         u32 revision_id;
 100         u32 abort;
 101         char data[0];
 102 };
 103
 104 struct shared_msr_entry {
 105         unsigned index;
 106         u64 data;
 107         u64 mask;
 108 };
 109
 110 struct vcpu_vmx {
 111         struct kvm_vcpu       vcpu;
 112         struct list_head      local_vcpus_link;
 113         unsigned long         host_rsp;
 114         int                   launched;
 115         u8                    fail;
 116         u32                   idt_vectoring_info;
 117         struct shared_msr_entry *guest_msrs;
 118         int                   nmsrs;
 119         int                   save_nmsrs;
 120 #ifdef CONFIG_X86_64
 121         u64                   msr_host_kernel_gs_base;
 122         u64                   msr_guest_kernel_gs_base;
 123 #endif
 124         struct vmcs          *vmcs;
 125         struct {
 126                 int           loaded;
 127                 u16           fs_sel, gs_sel, ldt_sel;
 128                 int           gs_ldt_reload_needed;
 129                 int           fs_reload_needed;
 130         } host_state;
 131         struct {
 132                 int vm86_active;
 133                 u8 save_iopl;
 134                 struct kvm_save_segment {
 135                         u16 selector;
 136                         unsigned long base;
 137                         u32 limit;
 138                         u32 ar;
 139                 } tr, es, ds, fs, gs;
 140                 struct {
 141                         bool pending;
 142                         u8 vector;
 143                         unsigned rip;
 144                 } irq;
 145         } rmode;
 146         int vpid;
 147         bool emulation_required;
 148
 149         /* Support for vnmi-less CPUs */
 150         int soft_vnmi_blocked;
 151         ktime_t entry_time;
 152         s64 vnmi_blocked_time;
 153         u32 exit_reason;
 154
 155         bool rdtscp_enabled;
 156 };
 157
 158 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
 159 {
 160         return container_of(vcpu, struct vcpu_vmx, vcpu);
 161 }
 162
 163 static int init_rmode(struct kvm *kvm);
 164 static u64 construct_eptp(unsigned long root_hpa);
 165
 166 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 167 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 168 static DEFINE_PER_CPU(struct list_head, vcpus_on_cpu);
 169
 170 static unsigned long *vmx_io_bitmap_a;
 171 static unsigned long *vmx_io_bitmap_b;
 172 static unsigned long *vmx_msr_bitmap_legacy;
 173 static unsigned long *vmx_msr_bitmap_longmode;
 174
 175 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 176 static DEFINE_SPINLOCK(vmx_vpid_lock);
 177
 178 static struct vmcs_config {
 179         int size;
 180         int order;
 181         u32 revision_id;
 182         u32 pin_based_exec_ctrl;
 183         u32 cpu_based_exec_ctrl;
 184         u32 cpu_based_2nd_exec_ctrl;
 185         u32 vmexit_ctrl;
 186         u32 vmentry_ctrl;
 187 } vmcs_config;
 188
 189 static struct vmx_capability {
 190         u32 ept;
 191         u32 vpid;
 192 } vmx_capability;
 193
 194 #define VMX_SEGMENT_FIELD(seg)                                  \
 195         [VCPU_SREG_##seg] = {                                   \
 196                 .selector = GUEST_##seg##_SELECTOR,             \
 197                 .base = GUEST_##seg##_BASE,                     \
 198                 .limit = GUEST_##seg##_LIMIT,                   \
 199                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
 200         }
 201
 202 static struct kvm_vmx_segment_field {
 203         unsigned selector;
 204         unsigned base;
 205         unsigned limit;
 206         unsigned ar_bytes;
 207 } kvm_vmx_segment_fields[] = {
 208         VMX_SEGMENT_FIELD(CS),
 209         VMX_SEGMENT_FIELD(DS),
 210         VMX_SEGMENT_FIELD(ES),
 211         VMX_SEGMENT_FIELD(FS),
 212         VMX_SEGMENT_FIELD(GS),
 213         VMX_SEGMENT_FIELD(SS),
 214         VMX_SEGMENT_FIELD(TR),
 215         VMX_SEGMENT_FIELD(LDTR),
 216 };
 217
 218 static u64 host_efer;
 219
 220 static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
 221
 222 /*
 223  * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
 224  * away by decrementing the array size.
 225  */
 226 static const u32 vmx_msr_index[] = {
 227 #ifdef CONFIG_X86_64
 228         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
 229 #endif
 230         MSR_EFER, MSR_TSC_AUX, MSR_K6_STAR,
 231 };
 232 #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
 233
 234 static inline int is_page_fault(u32 intr_info)
 235 {
 236         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 237                              INTR_INFO_VALID_MASK)) ==
 238                 (INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
 239 }
 240
 241 static inline int is_no_device(u32 intr_info)
 242 {
 243         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 244                              INTR_INFO_VALID_MASK)) ==
 245                 (INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
 246 }
 247
 248 static inline int is_invalid_opcode(u32 intr_info)
 249 {
 250         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 251                              INTR_INFO_VALID_MASK)) ==
 252                 (INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
 253 }
 254
 255 static inline int is_external_interrupt(u32 intr_info)
 256 {
 257         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
 258                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
 259 }
 260
 261 static inline int is_machine_check(u32 intr_info)
 262 {
 263         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 264                              INTR_INFO_VALID_MASK)) ==
 265                 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
 266 }
 267
 268 static inline int cpu_has_vmx_msr_bitmap(void)
 269 {
 270         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
 271 }
 272
 273 static inline int cpu_has_vmx_tpr_shadow(void)
 274 {
 275         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
 276 }
 277
 278 static inline int vm_need_tpr_shadow(struct kvm *kvm)
 279 {
 280         return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
 281 }
 282
 283 static inline int cpu_has_secondary_exec_ctrls(void)
 284 {
 285         return vmcs_config.cpu_based_exec_ctrl &
 286                 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
 287 }
 288
 289 static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
 290 {
 291         return vmcs_config.cpu_based_2nd_exec_ctrl &
 292                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
 293 }
 294
 295 static inline bool cpu_has_vmx_flexpriority(void)
 296 {
 297         return cpu_has_vmx_tpr_shadow() &&
 298                 cpu_has_vmx_virtualize_apic_accesses();
 299 }
 300
 301 static inline bool cpu_has_vmx_ept_execute_only(void)
 302 {
 303         return !!(vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT);
 304 }
 305
 306 static inline bool cpu_has_vmx_eptp_uncacheable(void)
 307 {
 308         return !!(vmx_capability.ept & VMX_EPTP_UC_BIT);
 309 }
 310
 311 static inline bool cpu_has_vmx_eptp_writeback(void)
 312 {
 313         return !!(vmx_capability.ept & VMX_EPTP_WB_BIT);
 314 }
 315
 316 static inline bool cpu_has_vmx_ept_2m_page(void)
 317 {
 318         return !!(vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT);
 319 }
 320
 321 static inline bool cpu_has_vmx_ept_1g_page(void)
 322 {
 323         return !!(vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT);
 324 }
 325
 326 static inline int cpu_has_vmx_invept_individual_addr(void)
 327 {
 328         return !!(vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT);
 329 }
 330
 331 static inline int cpu_has_vmx_invept_context(void)
 332 {
 333         return !!(vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT);
 334 }
 335
 336 static inline int cpu_has_vmx_invept_global(void)
 337 {
 338         return !!(vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT);
 339 }
 340
 341 static inline int cpu_has_vmx_ept(void)
 342 {
 343         return vmcs_config.cpu_based_2nd_exec_ctrl &
 344                 SECONDARY_EXEC_ENABLE_EPT;
 345 }
 346
 347 static inline int cpu_has_vmx_unrestricted_guest(void)
 348 {
 349         return vmcs_config.cpu_based_2nd_exec_ctrl &
 350                 SECONDARY_EXEC_UNRESTRICTED_GUEST;
 351 }
 352
 353 static inline int cpu_has_vmx_ple(void)
 354 {
 355         return vmcs_config.cpu_based_2nd_exec_ctrl &
 356                 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
 357 }
 358
 359 static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm)
 360 {
 361         return flexpriority_enabled &&
 362                 (cpu_has_vmx_virtualize_apic_accesses()) &&
 363                 (irqchip_in_kernel(kvm));
 364 }
 365
 366 static inline int cpu_has_vmx_vpid(void)
 367 {
 368         return vmcs_config.cpu_based_2nd_exec_ctrl &
 369                 SECONDARY_EXEC_ENABLE_VPID;
 370 }
 371
 372 static inline int cpu_has_vmx_rdtscp(void)
 373 {
 374         return vmcs_config.cpu_based_2nd_exec_ctrl &
 375                 SECONDARY_EXEC_RDTSCP;
 376 }
 377
 378 static inline int cpu_has_virtual_nmis(void)
 379 {
 380         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
 381 }
 382
 383 static inline bool report_flexpriority(void)
 384 {
 385         return flexpriority_enabled;
 386 }
 387
 388 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
 389 {
 390         int i;
 391
 392         for (i = 0; i < vmx->nmsrs; ++i)
 393                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
 394                         return i;
 395         return -1;
 396 }
 397
 398 static inline void __invvpid(int ext, u16 vpid, gva_t gva)
 399 {
 400     struct {
 401         u64 vpid : 16;
 402         u64 rsvd : 48;
 403         u64 gva;
 404     } operand = { vpid, 0, gva };
 405
 406     asm volatile (__ex(ASM_VMX_INVVPID)
 407                   /* CF==1 or ZF==1 --> rc = -1 */
 408                   "; ja 1f ; ud2 ; 1:"
 409                   : : "a"(&operand), "c"(ext) : "cc", "memory");
 410 }
 411
 412 static inline void __invept(int ext, u64 eptp, gpa_t gpa)
 413 {
 414         struct {
 415                 u64 eptp, gpa;
 416         } operand = {eptp, gpa};
 417
 418         asm volatile (__ex(ASM_VMX_INVEPT)
 419                         /* CF==1 or ZF==1 --> rc = -1 */
 420                         "; ja 1f ; ud2 ; 1:\n"
 421                         : : "a" (&operand), "c" (ext) : "cc", "memory");
 422 }
 423
 424 static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
 425 {
 426         int i;
 427
 428         i = __find_msr_index(vmx, msr);
 429         if (i >= 0)
 430                 return &vmx->guest_msrs[i];
 431         return NULL;
 432 }
 433
 434 static void vmcs_clear(struct vmcs *vmcs)
 435 {
 436         u64 phys_addr = __pa(vmcs);
 437         u8 error;
 438
 439         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
 440                       : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
 441                       : "cc", "memory");
 442         if (error)
 443                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
 444                        vmcs, phys_addr);
 445 }
 446
 447 static void __vcpu_clear(void *arg)
 448 {
 449         struct vcpu_vmx *vmx = arg;
 450         int cpu = raw_smp_processor_id();
 451
 452         if (vmx->vcpu.cpu == cpu)
 453                 vmcs_clear(vmx->vmcs);
 454         if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
 455                 per_cpu(current_vmcs, cpu) = NULL;
 456         rdtscll(vmx->vcpu.arch.host_tsc);
 457         list_del(&vmx->local_vcpus_link);
 458         vmx->vcpu.cpu = -1;
 459         vmx->launched = 0;
 460 }
 461
 462 static void vcpu_clear(struct vcpu_vmx *vmx)
 463 {
 464         if (vmx->vcpu.cpu == -1)
 465                 return;
 466         smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 1);
 467 }
 468
 469 static inline void vpid_sync_vcpu_all(struct vcpu_vmx *vmx)
 470 {
 471         if (vmx->vpid == 0)
 472                 return;
 473
 474         __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
 475 }
 476
 477 static inline void ept_sync_global(void)
 478 {
 479         if (cpu_has_vmx_invept_global())
 480                 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
 481 }
 482
 483 static inline void ept_sync_context(u64 eptp)
 484 {
 485         if (enable_ept) {
 486                 if (cpu_has_vmx_invept_context())
 487                         __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
 488                 else
 489                         ept_sync_global();
 490         }
 491 }
 492
 493 static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
 494 {
 495         if (enable_ept) {
 496                 if (cpu_has_vmx_invept_individual_addr())
 497                         __invept(VMX_EPT_EXTENT_INDIVIDUAL_ADDR,
 498                                         eptp, gpa);
 499                 else
 500                         ept_sync_context(eptp);
 501         }
 502 }
 503
 504 static unsigned long vmcs_readl(unsigned long field)
 505 {
 506         unsigned long value;
 507
 508         asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
 509                       : "=a"(value) : "d"(field) : "cc");
 510         return value;
 511 }
 512
 513 static u16 vmcs_read16(unsigned long field)
 514 {
 515         return vmcs_readl(field);
 516 }
 517
 518 static u32 vmcs_read32(unsigned long field)
 519 {
 520         return vmcs_readl(field);
 521 }
 522
 523 static u64 vmcs_read64(unsigned long field)
 524 {
 525 #ifdef CONFIG_X86_64
 526         return vmcs_readl(field);
 527 #else
 528         return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
 529 #endif
 530 }
 531
 532 static noinline void vmwrite_error(unsigned long field, unsigned long value)
 533 {
 534         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
 535                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
 536         dump_stack();
 537 }
 538
 539 static void vmcs_writel(unsigned long field, unsigned long value)
 540 {
 541         u8 error;
 542
 543         asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
 544                        : "=q"(error) : "a"(value), "d"(field) : "cc");
 545         if (unlikely(error))
 546                 vmwrite_error(field, value);
 547 }
 548
 549 static void vmcs_write16(unsigned long field, u16 value)
 550 {
 551         vmcs_writel(field, value);
 552 }
 553
 554 static void vmcs_write32(unsigned long field, u32 value)
 555 {
 556         vmcs_writel(field, value);
 557 }
 558
 559 static void vmcs_write64(unsigned long field, u64 value)
 560 {
 561         vmcs_writel(field, value);
 562 #ifndef CONFIG_X86_64
 563         asm volatile ("");
 564         vmcs_writel(field+1, value >> 32);
 565 #endif
 566 }
 567
 568 static void vmcs_clear_bits(unsigned long field, u32 mask)
 569 {
 570         vmcs_writel(field, vmcs_readl(field) & ~mask);
 571 }
 572
 573 static void vmcs_set_bits(unsigned long field, u32 mask)
 574 {
 575         vmcs_writel(field, vmcs_readl(field) | mask);
 576 }
 577
 578 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
 579 {
 580         u32 eb;
 581
 582         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR)
 583                 | (1u << NM_VECTOR);
 584         /*
 585          * Unconditionally intercept #DB so we can maintain dr6 without
 586          * reading it every exit.
 587          */
 588         eb |= 1u << DB_VECTOR;
 589         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
 590                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
 591                         eb |= 1u << BP_VECTOR;
 592         }
 593         if (to_vmx(vcpu)->rmode.vm86_active)
 594                 eb = ~0;
 595         if (enable_ept)
 596                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
 597         if (vcpu->fpu_active)
 598                 eb &= ~(1u << NM_VECTOR);
 599         vmcs_write32(EXCEPTION_BITMAP, eb);
 600 }
 601
 602 static void reload_tss(void)
 603 {
 604         /*
 605          * VT restores TR but not its size.  Useless.
 606          */
 607         struct descriptor_table gdt;
 608         struct desc_struct *descs;
 609
 610         kvm_get_gdt(&gdt);
 611         descs = (void *)gdt.base;
 612         descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
 613         load_TR_desc();
 614 }
 615
 616 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
 617 {
 618         u64 guest_efer;
 619         u64 ignore_bits;
 620
 621         guest_efer = vmx->vcpu.arch.shadow_efer;
 622
 623         /*
 624          * NX is emulated; LMA and LME handled by hardware; SCE meaninless
 625          * outside long mode
 626          */
 627         ignore_bits = EFER_NX | EFER_SCE;
 628 #ifdef CONFIG_X86_64
 629         ignore_bits |= EFER_LMA | EFER_LME;
 630         /* SCE is meaningful only in long mode on Intel */
 631         if (guest_efer & EFER_LMA)
 632                 ignore_bits &= ~(u64)EFER_SCE;
 633 #endif
 634         guest_efer &= ~ignore_bits;
 635         guest_efer |= host_efer & ignore_bits;
 636         vmx->guest_msrs[efer_offset].data = guest_efer;
 637         vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
 638         return true;
 639 }
 640
 641 static void vmx_save_host_state(struct kvm_vcpu *vcpu)
 642 {
 643         struct vcpu_vmx *vmx = to_vmx(vcpu);
 644         int i;
 645
 646         if (vmx->host_state.loaded)
 647                 return;
 648
 649         vmx->host_state.loaded = 1;
 650         /*
 651          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
 652          * allow segment selectors with cpl > 0 or ti == 1.
 653          */
 654         vmx->host_state.ldt_sel = kvm_read_ldt();
 655         vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
 656         vmx->host_state.fs_sel = kvm_read_fs();
 657         if (!(vmx->host_state.fs_sel & 7)) {
 658                 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
 659                 vmx->host_state.fs_reload_needed = 0;
 660         } else {
 661                 vmcs_write16(HOST_FS_SELECTOR, 0);
 662                 vmx->host_state.fs_reload_needed = 1;
 663         }
 664         vmx->host_state.gs_sel = kvm_read_gs();
 665         if (!(vmx->host_state.gs_sel & 7))
 666                 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
 667         else {
 668                 vmcs_write16(HOST_GS_SELECTOR, 0);
 669                 vmx->host_state.gs_ldt_reload_needed = 1;
 670         }
 671
 672 #ifdef CONFIG_X86_64
 673         vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
 674         vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
 675 #else
 676         vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
 677         vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
 678 #endif
 679
 680 #ifdef CONFIG_X86_64
 681         if (is_long_mode(&vmx->vcpu)) {
 682                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
 683                 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
 684         }
 685 #endif
 686         for (i = 0; i < vmx->save_nmsrs; ++i)
 687                 kvm_set_shared_msr(vmx->guest_msrs[i].index,
 688                                    vmx->guest_msrs[i].data,
 689                                    vmx->guest_msrs[i].mask);
 690 }
 691
 692 static void __vmx_load_host_state(struct vcpu_vmx *vmx)
 693 {
 694         unsigned long flags;
 695
 696         if (!vmx->host_state.loaded)
 697                 return;
 698
 699         ++vmx->vcpu.stat.host_state_reload;
 700         vmx->host_state.loaded = 0;
 701         if (vmx->host_state.fs_reload_needed)
 702                 kvm_load_fs(vmx->host_state.fs_sel);
 703         if (vmx->host_state.gs_ldt_reload_needed) {
 704                 kvm_load_ldt(vmx->host_state.ldt_sel);
 705                 /*
 706                  * If we have to reload gs, we must take care to
 707                  * preserve our gs base.
 708                  */
 709                 local_irq_save(flags);
 710                 kvm_load_gs(vmx->host_state.gs_sel);
 711 #ifdef CONFIG_X86_64
 712                 wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
 713 #endif
 714                 local_irq_restore(flags);
 715         }
 716         reload_tss();
 717 #ifdef CONFIG_X86_64
 718         if (is_long_mode(&vmx->vcpu)) {
 719                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
 720                 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
 721         }
 722 #endif
 723 }
 724
 725 static void vmx_load_host_state(struct vcpu_vmx *vmx)
 726 {
 727         preempt_disable();
 728         __vmx_load_host_state(vmx);
 729         preempt_enable();
 730 }
 731
 732 /*
 733  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 734  * vcpu mutex is already taken.
 735  */
 736 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 737 {
 738         struct vcpu_vmx *vmx = to_vmx(vcpu);
 739         u64 phys_addr = __pa(vmx->vmcs);
 740         u64 tsc_this, delta, new_offset;
 741
 742         if (vcpu->cpu != cpu) {
 743                 vcpu_clear(vmx);
 744                 kvm_migrate_timers(vcpu);
 745                 set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests);
 746                 local_irq_disable();
 747                 list_add(&vmx->local_vcpus_link,
 748                          &per_cpu(vcpus_on_cpu, cpu));
 749                 local_irq_enable();
 750         }
 751
 752         if (per_cpu(current_vmcs, cpu) != vmx->vmcs) {
 753                 u8 error;
 754
 755                 per_cpu(current_vmcs, cpu) = vmx->vmcs;
 756                 asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
 757                               : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
 758                               : "cc");
 759                 if (error)
 760                         printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
 761                                vmx->vmcs, phys_addr);
 762         }
 763
 764         if (vcpu->cpu != cpu) {
 765                 struct descriptor_table dt;
 766                 unsigned long sysenter_esp;
 767
 768                 vcpu->cpu = cpu;
 769                 /*
 770                  * Linux uses per-cpu TSS and GDT, so set these when switching
 771                  * processors.
 772                  */
 773                 vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
 774                 kvm_get_gdt(&dt);
 775                 vmcs_writel(HOST_GDTR_BASE, dt.base);   /* 22.2.4 */
 776
 777                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
 778                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
 779
 780                 /*
 781                  * Make sure the time stamp counter is monotonous.
 782                  */
 783                 rdtscll(tsc_this);
 784                 if (tsc_this < vcpu->arch.host_tsc) {
 785                         delta = vcpu->arch.host_tsc - tsc_this;
 786                         new_offset = vmcs_read64(TSC_OFFSET) + delta;
 787                         vmcs_write64(TSC_OFFSET, new_offset);
 788                 }
 789         }
 790 }
 791
 792 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
 793 {
 794         __vmx_load_host_state(to_vmx(vcpu));
 795 }
 796
 797 static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
 798 {
 799         if (vcpu->fpu_active)
 800                 return;
 801         vcpu->fpu_active = 1;
 802         vmcs_clear_bits(GUEST_CR0, X86_CR0_TS);
 803         if (kvm_read_cr0_bits(vcpu, X86_CR0_TS))
 804                 vmcs_set_bits(GUEST_CR0, X86_CR0_TS);
 805         update_exception_bitmap(vcpu);
 806         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
 807         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
 808 }
 809
 810 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
 811
 812 static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
 813 {
 814         vmx_decache_cr0_guest_bits(vcpu);
 815         vmcs_set_bits(GUEST_CR0, X86_CR0_TS);
 816         update_exception_bitmap(vcpu);
 817         vcpu->arch.cr0_guest_owned_bits = 0;
 818         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
 819         vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
 820 }
 821
 822 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
 823 {
 824         unsigned long rflags;
 825
 826         rflags = vmcs_readl(GUEST_RFLAGS);
 827         if (to_vmx(vcpu)->rmode.vm86_active)
 828                 rflags &= ~(unsigned long)(X86_EFLAGS_IOPL | X86_EFLAGS_VM);
 829         return rflags;
 830 }
 831
 832 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 833 {
 834         if (to_vmx(vcpu)->rmode.vm86_active)
 835                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
 836         vmcs_writel(GUEST_RFLAGS, rflags);
 837 }
 838
 839 static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 840 {
 841         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
 842         int ret = 0;
 843
 844         if (interruptibility & GUEST_INTR_STATE_STI)
 845                 ret |= X86_SHADOW_INT_STI;
 846         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
 847                 ret |= X86_SHADOW_INT_MOV_SS;
 848
 849         return ret & mask;
 850 }
 851
 852 static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 853 {
 854         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
 855         u32 interruptibility = interruptibility_old;
 856
 857         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
 858
 859         if (mask & X86_SHADOW_INT_MOV_SS)
 860                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
 861         if (mask & X86_SHADOW_INT_STI)
 862                 interruptibility |= GUEST_INTR_STATE_STI;
 863
 864         if ((interruptibility != interruptibility_old))
 865                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
 866 }
 867
 868 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 869 {
 870         unsigned long rip;
 871
 872         rip = kvm_rip_read(vcpu);
 873         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
 874         kvm_rip_write(vcpu, rip);
 875
 876         /* skipping an emulated instruction also counts */
 877         vmx_set_interrupt_shadow(vcpu, 0);
 878 }
 879
 880 static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
 881                                 bool has_error_code, u32 error_code)
 882 {
 883         struct vcpu_vmx *vmx = to_vmx(vcpu);
 884         u32 intr_info = nr | INTR_INFO_VALID_MASK;
 885
 886         if (has_error_code) {
 887                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
 888                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
 889         }
 890
 891         if (vmx->rmode.vm86_active) {
 892                 vmx->rmode.irq.pending = true;
 893                 vmx->rmode.irq.vector = nr;
 894                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
 895                 if (kvm_exception_is_soft(nr))
 896                         vmx->rmode.irq.rip +=
 897                                 vmx->vcpu.arch.event_exit_inst_len;
 898                 intr_info |= INTR_TYPE_SOFT_INTR;
 899                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
 900                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
 901                 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
 902                 return;
 903         }
 904
 905         if (kvm_exception_is_soft(nr)) {
 906                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
 907                              vmx->vcpu.arch.event_exit_inst_len);
 908                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
 909         } else
 910                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
 911
 912         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
 913 }
 914
 915 static bool vmx_rdtscp_supported(void)
 916 {
 917         return cpu_has_vmx_rdtscp();
 918 }
 919
 920 /*
 921  * Swap MSR entry in host/guest MSR entry array.
 922  */
 923 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
 924 {
 925         struct shared_msr_entry tmp;
 926
 927         tmp = vmx->guest_msrs[to];
 928         vmx->guest_msrs[to] = vmx->guest_msrs[from];
 929         vmx->guest_msrs[from] = tmp;
 930 }
 931
 932 /*
 933  * Set up the vmcs to automatically save and restore system
 934  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 935  * mode, as fiddling with msrs is very expensive.
 936  */
 937 static void setup_msrs(struct vcpu_vmx *vmx)
 938 {
 939         int save_nmsrs, index;
 940         unsigned long *msr_bitmap;
 941
 942         vmx_load_host_state(vmx);
 943         save_nmsrs = 0;
 944 #ifdef CONFIG_X86_64
 945         if (is_long_mode(&vmx->vcpu)) {
 946                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
 947                 if (index >= 0)
 948                         move_msr_up(vmx, index, save_nmsrs++);
 949                 index = __find_msr_index(vmx, MSR_LSTAR);
 950                 if (index >= 0)
 951                         move_msr_up(vmx, index, save_nmsrs++);
 952                 index = __find_msr_index(vmx, MSR_CSTAR);
 953                 if (index >= 0)
 954                         move_msr_up(vmx, index, save_nmsrs++);
 955                 index = __find_msr_index(vmx, MSR_TSC_AUX);
 956                 if (index >= 0 && vmx->rdtscp_enabled)
 957                         move_msr_up(vmx, index, save_nmsrs++);
 958                 /*
 959                  * MSR_K6_STAR is only needed on long mode guests, and only
 960                  * if efer.sce is enabled.
 961                  */
 962                 index = __find_msr_index(vmx, MSR_K6_STAR);
 963                 if ((index >= 0) && (vmx->vcpu.arch.shadow_efer & EFER_SCE))
 964                         move_msr_up(vmx, index, save_nmsrs++);
 965         }
 966 #endif
 967         index = __find_msr_index(vmx, MSR_EFER);
 968         if (index >= 0 && update_transition_efer(vmx, index))
 969                 move_msr_up(vmx, index, save_nmsrs++);
 970
 971         vmx->save_nmsrs = save_nmsrs;
 972
 973         if (cpu_has_vmx_msr_bitmap()) {
 974                 if (is_long_mode(&vmx->vcpu))
 975                         msr_bitmap = vmx_msr_bitmap_longmode;
 976                 else
 977                         msr_bitmap = vmx_msr_bitmap_legacy;
 978
 979                 vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
 980         }
 981 }
 982
 983 /*
 984  * reads and returns guest's timestamp counter "register"
 985  * guest_tsc = host_tsc + tsc_offset    -- 21.3
 986  */
 987 static u64 guest_read_tsc(void)
 988 {
 989         u64 host_tsc, tsc_offset;
 990
 991         rdtscll(host_tsc);
 992         tsc_offset = vmcs_read64(TSC_OFFSET);
 993         return host_tsc + tsc_offset;
 994 }
 995
 996 /*
 997  * writes 'guest_tsc' into guest's timestamp counter "register"
 998  * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
 999  */
1000 static void guest_write_tsc(u64 guest_tsc, u64 host_tsc)
1001 {
1002         vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
1003 }
1004
1005 /*
1006  * Reads an msr value (of 'msr_index') into 'pdata'.
1007  * Returns 0 on success, non-0 otherwise.
1008  * Assumes vcpu_load() was already called.
1009  */
1010 static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1011 {
1012         u64 data;
1013         struct shared_msr_entry *msr;
1014
1015         if (!pdata) {
1016                 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
1017                 return -EINVAL;
1018         }
1019
1020         switch (msr_index) {
1021 #ifdef CONFIG_X86_64
1022         case MSR_FS_BASE:
1023                 data = vmcs_readl(GUEST_FS_BASE);
1024                 break;
1025         case MSR_GS_BASE:
1026                 data = vmcs_readl(GUEST_GS_BASE);
1027                 break;
1028         case MSR_KERNEL_GS_BASE:
1029                 vmx_load_host_state(to_vmx(vcpu));
1030                 data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
1031                 break;
1032 #endif
1033         case MSR_EFER:
1034                 return kvm_get_msr_common(vcpu, msr_index, pdata);
1035         case MSR_IA32_TSC:
1036                 data = guest_read_tsc();
1037                 break;
1038         case MSR_IA32_SYSENTER_CS:
1039                 data = vmcs_read32(GUEST_SYSENTER_CS);
1040                 break;
1041         case MSR_IA32_SYSENTER_EIP:
1042                 data = vmcs_readl(GUEST_SYSENTER_EIP);
1043                 break;
1044         case MSR_IA32_SYSENTER_ESP:
1045                 data = vmcs_readl(GUEST_SYSENTER_ESP);
1046                 break;
1047         case MSR_TSC_AUX:
1048                 if (!to_vmx(vcpu)->rdtscp_enabled)
1049                         return 1;
1050                 /* Otherwise falls through */
1051         default:
1052                 vmx_load_host_state(to_vmx(vcpu));
1053                 msr = find_msr_entry(to_vmx(vcpu), msr_index);
1054                 if (msr) {
1055                         vmx_load_host_state(to_vmx(vcpu));
1056                         data = msr->data;
1057                         break;
1058                 }
1059                 return kvm_get_msr_common(vcpu, msr_index, pdata);
1060         }
1061
1062         *pdata = data;
1063         return 0;
1064 }
1065
1066 /*
1067  * Writes msr value into into the appropriate "register".
1068  * Returns 0 on success, non-0 otherwise.
1069  * Assumes vcpu_load() was already called.
1070  */
1071 static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1072 {
1073         struct vcpu_vmx *vmx = to_vmx(vcpu);
1074         struct shared_msr_entry *msr;
1075         u64 host_tsc;
1076         int ret = 0;
1077
1078         switch (msr_index) {
1079         case MSR_EFER:
1080                 vmx_load_host_state(vmx);
1081                 ret = kvm_set_msr_common(vcpu, msr_index, data);
1082                 break;
1083 #ifdef CONFIG_X86_64
1084         case MSR_FS_BASE:
1085                 vmcs_writel(GUEST_FS_BASE, data);
1086                 break;
1087         case MSR_GS_BASE:
1088                 vmcs_writel(GUEST_GS_BASE, data);
1089                 break;
1090         case MSR_KERNEL_GS_BASE:
1091                 vmx_load_host_state(vmx);
1092                 vmx->msr_guest_kernel_gs_base = data;
1093                 break;
1094 #endif
1095         case MSR_IA32_SYSENTER_CS:
1096                 vmcs_write32(GUEST_SYSENTER_CS, data);
1097                 break;
1098         case MSR_IA32_SYSENTER_EIP:
1099                 vmcs_writel(GUEST_SYSENTER_EIP, data);
1100                 break;
1101         case MSR_IA32_SYSENTER_ESP:
1102                 vmcs_writel(GUEST_SYSENTER_ESP, data);
1103                 break;
1104         case MSR_IA32_TSC:
1105                 rdtscll(host_tsc);
1106                 guest_write_tsc(data, host_tsc);
1107                 break;
1108         case MSR_IA32_CR_PAT:
1109                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
1110                         vmcs_write64(GUEST_IA32_PAT, data);
1111                         vcpu->arch.pat = data;
1112                         break;
1113                 }
1114                 ret = kvm_set_msr_common(vcpu, msr_index, data);
1115                 break;
1116         case MSR_TSC_AUX:
1117                 if (!vmx->rdtscp_enabled)
1118                         return 1;
1119                 /* Check reserved bit, higher 32 bits should be zero */
1120                 if ((data >> 32) != 0)
1121                         return 1;
1122                 /* Otherwise falls through */
1123         default:
1124                 msr = find_msr_entry(vmx, msr_index);
1125                 if (msr) {
1126                         vmx_load_host_state(vmx);
1127                         msr->data = data;
1128                         break;
1129                 }
1130                 ret = kvm_set_msr_common(vcpu, msr_index, data);
1131         }
1132
1133         return ret;
1134 }
1135
1136 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1137 {
1138         __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
1139         switch (reg) {
1140         case VCPU_REGS_RSP:
1141                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
1142                 break;
1143         case VCPU_REGS_RIP:
1144                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
1145                 break;
1146         case VCPU_EXREG_PDPTR:
1147                 if (enable_ept)
1148                         ept_save_pdptrs(vcpu);
1149                 break;
1150         default:
1151                 break;
1152         }
1153 }
1154
1155 static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
1156 {
1157         if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1158                 vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
1159         else
1160                 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
1161
1162         update_exception_bitmap(vcpu);
1163 }
1164
1165 static __init int cpu_has_kvm_support(void)
1166 {
1167         return cpu_has_vmx();
1168 }
1169
1170 static __init int vmx_disabled_by_bios(void)
1171 {
1172         u64 msr;
1173
1174         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
1175         return (msr & (FEATURE_CONTROL_LOCKED |
1176                        FEATURE_CONTROL_VMXON_ENABLED))
1177             == FEATURE_CONTROL_LOCKED;
1178         /* locked but not enabled */
1179 }
1180
1181 static int hardware_enable(void *garbage)
1182 {
1183         int cpu = raw_smp_processor_id();
1184         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
1185         u64 old;
1186
1187         if (read_cr4() & X86_CR4_VMXE)
1188                 return -EBUSY;
1189
1190         INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu));
1191         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
1192         if ((old & (FEATURE_CONTROL_LOCKED |
1193                     FEATURE_CONTROL_VMXON_ENABLED))
1194             != (FEATURE_CONTROL_LOCKED |
1195                 FEATURE_CONTROL_VMXON_ENABLED))
1196                 /* enable and lock */
1197                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old |
1198                        FEATURE_CONTROL_LOCKED |
1199                        FEATURE_CONTROL_VMXON_ENABLED);
1200         write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
1201         asm volatile (ASM_VMX_VMXON_RAX
1202                       : : "a"(&phys_addr), "m"(phys_addr)
1203                       : "memory", "cc");
1204
1205         ept_sync_global();
1206
1207         return 0;
1208 }
1209
1210 static void vmclear_local_vcpus(void)
1211 {
1212         int cpu = raw_smp_processor_id();
1213         struct vcpu_vmx *vmx, *n;
1214
1215         list_for_each_entry_safe(vmx, n, &per_cpu(vcpus_on_cpu, cpu),
1216                                  local_vcpus_link)
1217                 __vcpu_clear(vmx);
1218 }
1219
1220
1221 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
1222  * tricks.
1223  */
1224 static void kvm_cpu_vmxoff(void)
1225 {
1226         asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
1227         write_cr4(read_cr4() & ~X86_CR4_VMXE);
1228 }
1229
1230 static void hardware_disable(void *garbage)
1231 {
1232         vmclear_local_vcpus();
1233         kvm_cpu_vmxoff();
1234 }
1235
1236 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
1237                                       u32 msr, u32 *result)
1238 {
1239         u32 vmx_msr_low, vmx_msr_high;
1240         u32 ctl = ctl_min | ctl_opt;
1241
1242         rdmsr(msr, vmx_msr_low, vmx_msr_high);
1243
1244         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
1245         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
1246
1247         /* Ensure minimum (required) set of control bits are supported. */
1248         if (ctl_min & ~ctl)
1249                 return -EIO;
1250
1251         *result = ctl;
1252         return 0;
1253 }
1254
1255 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
1256 {
1257         u32 vmx_msr_low, vmx_msr_high;
1258         u32 min, opt, min2, opt2;
1259         u32 _pin_based_exec_control = 0;
1260         u32 _cpu_based_exec_control = 0;
1261         u32 _cpu_based_2nd_exec_control = 0;
1262         u32 _vmexit_control = 0;
1263         u32 _vmentry_control = 0;
1264
1265         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
1266         opt = PIN_BASED_VIRTUAL_NMIS;
1267         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
1268                                 &_pin_based_exec_control) < 0)
1269                 return -EIO;
1270
1271         min = CPU_BASED_HLT_EXITING |
1272 #ifdef CONFIG_X86_64
1273               CPU_BASED_CR8_LOAD_EXITING |
1274               CPU_BASED_CR8_STORE_EXITING |
1275 #endif
1276               CPU_BASED_CR3_LOAD_EXITING |
1277               CPU_BASED_CR3_STORE_EXITING |
1278               CPU_BASED_USE_IO_BITMAPS |
1279               CPU_BASED_MOV_DR_EXITING |
1280               CPU_BASED_USE_TSC_OFFSETING |
1281               CPU_BASED_MWAIT_EXITING |
1282               CPU_BASED_MONITOR_EXITING |
1283               CPU_BASED_INVLPG_EXITING;
1284         opt = CPU_BASED_TPR_SHADOW |
1285               CPU_BASED_USE_MSR_BITMAPS |
1286               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1287         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
1288                                 &_cpu_based_exec_control) < 0)
1289                 return -EIO;
1290 #ifdef CONFIG_X86_64
1291         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
1292                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
1293                                            ~CPU_BASED_CR8_STORE_EXITING;
1294 #endif
1295         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
1296                 min2 = 0;
1297                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
1298                         SECONDARY_EXEC_WBINVD_EXITING |
1299                         SECONDARY_EXEC_ENABLE_VPID |
1300                         SECONDARY_EXEC_ENABLE_EPT |
1301                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
1302                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
1303                         SECONDARY_EXEC_RDTSCP;
1304                 if (adjust_vmx_controls(min2, opt2,
1305                                         MSR_IA32_VMX_PROCBASED_CTLS2,
1306                                         &_cpu_based_2nd_exec_control) < 0)
1307                         return -EIO;
1308         }
1309 #ifndef CONFIG_X86_64
1310         if (!(_cpu_based_2nd_exec_control &
1311                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
1312                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
1313 #endif
1314         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
1315                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
1316                    enabled */
1317                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
1318                                              CPU_BASED_CR3_STORE_EXITING |
1319                                              CPU_BASED_INVLPG_EXITING);
1320                 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
1321                       vmx_capability.ept, vmx_capability.vpid);
1322         }
1323
1324         min = 0;
1325 #ifdef CONFIG_X86_64
1326         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
1327 #endif
1328         opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT;
1329         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
1330                                 &_vmexit_control) < 0)
1331                 return -EIO;
1332
1333         min = 0;
1334         opt = VM_ENTRY_LOAD_IA32_PAT;
1335         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
1336                                 &_vmentry_control) < 0)
1337                 return -EIO;
1338
1339         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
1340
1341         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
1342         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
1343                 return -EIO;
1344
1345 #ifdef CONFIG_X86_64
1346         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
1347         if (vmx_msr_high & (1u<<16))
1348                 return -EIO;
1349 #endif
1350
1351         /* Require Write-Back (WB) memory type for VMCS accesses. */
1352         if (((vmx_msr_high >> 18) & 15) != 6)
1353                 return -EIO;
1354
1355         vmcs_conf->size = vmx_msr_high & 0x1fff;
1356         vmcs_conf->order = get_order(vmcs_config.size);
1357         vmcs_conf->revision_id = vmx_msr_low;
1358
1359         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
1360         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
1361         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
1362         vmcs_conf->vmexit_ctrl         = _vmexit_control;
1363         vmcs_conf->vmentry_ctrl        = _vmentry_control;
1364
1365         return 0;
1366 }
1367
1368 static struct vmcs *alloc_vmcs_cpu(int cpu)
1369 {
1370         int node = cpu_to_node(cpu);
1371         struct page *pages;
1372         struct vmcs *vmcs;
1373
1374         pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
1375         if (!pages)
1376                 return NULL;
1377         vmcs = page_address(pages);
1378         memset(vmcs, 0, vmcs_config.size);
1379         vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
1380         return vmcs;
1381 }
1382
1383 static struct vmcs *alloc_vmcs(void)
1384 {
1385         return alloc_vmcs_cpu(raw_smp_processor_id());
1386 }
1387
1388 static void free_vmcs(struct vmcs *vmcs)
1389 {
1390         free_pages((unsigned long)vmcs, vmcs_config.order);
1391 }
1392
1393 static void free_kvm_area(void)
1394 {
1395         int cpu;
1396
1397         for_each_possible_cpu(cpu) {
1398                 free_vmcs(per_cpu(vmxarea, cpu));
1399                 per_cpu(vmxarea, cpu) = NULL;
1400         }
1401 }
1402
1403 static __init int alloc_kvm_area(void)
1404 {
1405         int cpu;
1406
1407         for_each_possible_cpu(cpu) {
1408                 struct vmcs *vmcs;
1409
1410                 vmcs = alloc_vmcs_cpu(cpu);
1411                 if (!vmcs) {
1412                         free_kvm_area();
1413                         return -ENOMEM;
1414                 }
1415
1416                 per_cpu(vmxarea, cpu) = vmcs;
1417         }
1418         return 0;
1419 }
1420
1421 static __init int hardware_setup(void)
1422 {
1423         if (setup_vmcs_config(&vmcs_config) < 0)
1424                 return -EIO;
1425
1426         if (boot_cpu_has(X86_FEATURE_NX))
1427                 kvm_enable_efer_bits(EFER_NX);
1428
1429         if (!cpu_has_vmx_vpid())
1430                 enable_vpid = 0;
1431
1432         if (!cpu_has_vmx_ept()) {
1433                 enable_ept = 0;
1434                 enable_unrestricted_guest = 0;
1435         }
1436
1437         if (!cpu_has_vmx_unrestricted_guest())
1438                 enable_unrestricted_guest = 0;
1439
1440         if (!cpu_has_vmx_flexpriority())
1441                 flexpriority_enabled = 0;
1442
1443         if (!cpu_has_vmx_tpr_shadow())
1444                 kvm_x86_ops->update_cr8_intercept = NULL;
1445
1446         if (enable_ept && !cpu_has_vmx_ept_2m_page())
1447                 kvm_disable_largepages();
1448
1449         if (!cpu_has_vmx_ple())
1450                 ple_gap = 0;
1451
1452         return alloc_kvm_area();
1453 }
1454
1455 static __exit void hardware_unsetup(void)
1456 {
1457         free_kvm_area();
1458 }
1459
1460 static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
1461 {
1462         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1463
1464         if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
1465                 vmcs_write16(sf->selector, save->selector);
1466                 vmcs_writel(sf->base, save->base);
1467                 vmcs_write32(sf->limit, save->limit);
1468                 vmcs_write32(sf->ar_bytes, save->ar);
1469         } else {
1470                 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
1471                         << AR_DPL_SHIFT;
1472                 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
1473         }
1474 }
1475
1476 static void enter_pmode(struct kvm_vcpu *vcpu)
1477 {
1478         unsigned long flags;
1479         struct vcpu_vmx *vmx = to_vmx(vcpu);
1480
1481         vmx->emulation_required = 1;
1482         vmx->rmode.vm86_active = 0;
1483
1484         vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
1485         vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
1486         vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
1487
1488         flags = vmcs_readl(GUEST_RFLAGS);
1489         flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM);
1490         flags |= (vmx->rmode.save_iopl << IOPL_SHIFT);
1491         vmcs_writel(GUEST_RFLAGS, flags);
1492
1493         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
1494                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
1495
1496         update_exception_bitmap(vcpu);
1497
1498         if (emulate_invalid_guest_state)
1499                 return;
1500
1501         fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es);
1502         fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds);
1503         fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs);
1504         fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs);
1505
1506         vmcs_write16(GUEST_SS_SELECTOR, 0);
1507         vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
1508
1509         vmcs_write16(GUEST_CS_SELECTOR,
1510                      vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
1511         vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1512 }
1513
1514 static gva_t rmode_tss_base(struct kvm *kvm)
1515 {
1516         if (!kvm->arch.tss_addr) {
1517                 struct kvm_memslots *slots;
1518                 gfn_t base_gfn;
1519
1520                 slots = rcu_dereference(kvm->memslots);
1521                 base_gfn = kvm->memslots->memslots[0].base_gfn +
1522                                  kvm->memslots->memslots[0].npages - 3;
1523                 return base_gfn << PAGE_SHIFT;
1524         }
1525         return kvm->arch.tss_addr;
1526 }
1527
1528 static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
1529 {
1530         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1531
1532         save->selector = vmcs_read16(sf->selector);
1533         save->base = vmcs_readl(sf->base);
1534         save->limit = vmcs_read32(sf->limit);
1535         save->ar = vmcs_read32(sf->ar_bytes);
1536         vmcs_write16(sf->selector, save->base >> 4);
1537         vmcs_write32(sf->base, save->base & 0xfffff);
1538         vmcs_write32(sf->limit, 0xffff);
1539         vmcs_write32(sf->ar_bytes, 0xf3);
1540 }
1541
1542 static void enter_rmode(struct kvm_vcpu *vcpu)
1543 {
1544         unsigned long flags;
1545         struct vcpu_vmx *vmx = to_vmx(vcpu);
1546
1547         if (enable_unrestricted_guest)
1548                 return;
1549
1550         vmx->emulation_required = 1;
1551         vmx->rmode.vm86_active = 1;
1552
1553         vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
1554         vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
1555
1556         vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
1557         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
1558
1559         vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
1560         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1561
1562         flags = vmcs_readl(GUEST_RFLAGS);
1563         vmx->rmode.save_iopl
1564                 = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1565
1566         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
1567
1568         vmcs_writel(GUEST_RFLAGS, flags);
1569         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
1570         update_exception_bitmap(vcpu);
1571
1572         if (emulate_invalid_guest_state)
1573                 goto continue_rmode;
1574
1575         vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
1576         vmcs_write32(GUEST_SS_LIMIT, 0xffff);
1577         vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
1578
1579         vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
1580         vmcs_write32(GUEST_CS_LIMIT, 0xffff);
1581         if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
1582                 vmcs_writel(GUEST_CS_BASE, 0xf0000);
1583         vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
1584
1585         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
1586         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
1587         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
1588         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs);
1589
1590 continue_rmode:
1591         kvm_mmu_reset_context(vcpu);
1592         init_rmode(vcpu->kvm);
1593 }
1594
1595 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
1596 {
1597         struct vcpu_vmx *vmx = to_vmx(vcpu);
1598         struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
1599
1600         if (!msr)
1601                 return;
1602
1603         /*
1604          * Force kernel_gs_base reloading before EFER changes, as control
1605          * of this msr depends on is_long_mode().
1606          */
1607         vmx_load_host_state(to_vmx(vcpu));
1608         vcpu->arch.shadow_efer = efer;
1609         if (!msr)
1610                 return;
1611         if (efer & EFER_LMA) {
1612                 vmcs_write32(VM_ENTRY_CONTROLS,
1613                              vmcs_read32(VM_ENTRY_CONTROLS) |
1614                              VM_ENTRY_IA32E_MODE);
1615                 msr->data = efer;
1616         } else {
1617                 vmcs_write32(VM_ENTRY_CONTROLS,
1618                              vmcs_read32(VM_ENTRY_CONTROLS) &
1619                              ~VM_ENTRY_IA32E_MODE);
1620
1621                 msr->data = efer & ~EFER_LME;
1622         }
1623         setup_msrs(vmx);
1624 }
1625
1626 #ifdef CONFIG_X86_64
1627
1628 static void enter_lmode(struct kvm_vcpu *vcpu)
1629 {
1630         u32 guest_tr_ar;
1631
1632         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
1633         if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
1634                 printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
1635                        __func__);
1636                 vmcs_write32(GUEST_TR_AR_BYTES,
1637                              (guest_tr_ar & ~AR_TYPE_MASK)
1638                              | AR_TYPE_BUSY_64_TSS);
1639         }
1640         vcpu->arch.shadow_efer |= EFER_LMA;
1641         vmx_set_efer(vcpu, vcpu->arch.shadow_efer);
1642 }
1643
1644 static void exit_lmode(struct kvm_vcpu *vcpu)
1645 {
1646         vcpu->arch.shadow_efer &= ~EFER_LMA;
1647
1648         vmcs_write32(VM_ENTRY_CONTROLS,
1649                      vmcs_read32(VM_ENTRY_CONTROLS)
1650                      & ~VM_ENTRY_IA32E_MODE);
1651 }
1652
1653 #endif
1654
1655 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
1656 {
1657         vpid_sync_vcpu_all(to_vmx(vcpu));
1658         if (enable_ept)
1659                 ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
1660 }
1661
1662 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1663 {
1664         ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
1665
1666         vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
1667         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
1668 }
1669
1670 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1671 {
1672         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
1673
1674         vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
1675         vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
1676 }
1677
1678 static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
1679 {
1680         if (!test_bit(VCPU_EXREG_PDPTR,
1681                       (unsigned long *)&vcpu->arch.regs_dirty))
1682                 return;
1683
1684         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1685                 vmcs_write64(GUEST_PDPTR0, vcpu->arch.pdptrs[0]);
1686                 vmcs_write64(GUEST_PDPTR1, vcpu->arch.pdptrs[1]);
1687                 vmcs_write64(GUEST_PDPTR2, vcpu->arch.pdptrs[2]);
1688                 vmcs_write64(GUEST_PDPTR3, vcpu->arch.pdptrs[3]);
1689         }
1690 }
1691
1692 static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
1693 {
1694         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1695                 vcpu->arch.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
1696                 vcpu->arch.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
1697                 vcpu->arch.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
1698                 vcpu->arch.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
1699         }
1700
1701         __set_bit(VCPU_EXREG_PDPTR,
1702                   (unsigned long *)&vcpu->arch.regs_avail);
1703         __set_bit(VCPU_EXREG_PDPTR,
1704                   (unsigned long *)&vcpu->arch.regs_dirty);
1705 }
1706
1707 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
1708
1709 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
1710                                         unsigned long cr0,
1711                                         struct kvm_vcpu *vcpu)
1712 {
1713         if (!(cr0 & X86_CR0_PG)) {
1714                 /* From paging/starting to nonpaging */
1715                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
1716                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
1717                              (CPU_BASED_CR3_LOAD_EXITING |
1718                               CPU_BASED_CR3_STORE_EXITING));
1719                 vcpu->arch.cr0 = cr0;
1720                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1721         } else if (!is_paging(vcpu)) {
1722                 /* From nonpaging to paging */
1723                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
1724                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
1725                              ~(CPU_BASED_CR3_LOAD_EXITING |
1726                                CPU_BASED_CR3_STORE_EXITING));
1727                 vcpu->arch.cr0 = cr0;
1728                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1729         }
1730
1731         if (!(cr0 & X86_CR0_WP))
1732                 *hw_cr0 &= ~X86_CR0_WP;
1733 }
1734
1735 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1736 {
1737         struct vcpu_vmx *vmx = to_vmx(vcpu);
1738         unsigned long hw_cr0;
1739
1740         if (enable_unrestricted_guest)
1741                 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
1742                         | KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
1743         else
1744                 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
1745
1746         if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
1747                 enter_pmode(vcpu);
1748
1749         if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
1750                 enter_rmode(vcpu);
1751
1752 #ifdef CONFIG_X86_64
1753         if (vcpu->arch.shadow_efer & EFER_LME) {
1754                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
1755                         enter_lmode(vcpu);
1756                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
1757                         exit_lmode(vcpu);
1758         }
1759 #endif
1760
1761         if (enable_ept)
1762                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
1763
1764         if (!vcpu->fpu_active)
1765                 hw_cr0 |= X86_CR0_TS;
1766
1767         vmcs_writel(CR0_READ_SHADOW, cr0);
1768         vmcs_writel(GUEST_CR0, hw_cr0);
1769         vcpu->arch.cr0 = cr0;
1770 }
1771
1772 static u64 construct_eptp(unsigned long root_hpa)
1773 {
1774         u64 eptp;
1775
1776         /* TODO write the value reading from MSR */
1777         eptp = VMX_EPT_DEFAULT_MT |
1778                 VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
1779         eptp |= (root_hpa & PAGE_MASK);
1780
1781         return eptp;
1782 }
1783
1784 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1785 {
1786         unsigned long guest_cr3;
1787         u64 eptp;
1788
1789         guest_cr3 = cr3;
1790         if (enable_ept) {
1791                 eptp = construct_eptp(cr3);
1792                 vmcs_write64(EPT_POINTER, eptp);
1793                 guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 :
1794                         vcpu->kvm->arch.ept_identity_map_addr;
1795                 ept_load_pdptrs(vcpu);
1796         }
1797
1798         vmx_flush_tlb(vcpu);
1799         vmcs_writel(GUEST_CR3, guest_cr3);
1800 }
1801
1802 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1803 {
1804         unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
1805                     KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
1806
1807         vcpu->arch.cr4 = cr4;
1808         if (enable_ept) {
1809                 if (!is_paging(vcpu)) {
1810                         hw_cr4 &= ~X86_CR4_PAE;
1811                         hw_cr4 |= X86_CR4_PSE;
1812                 } else if (!(cr4 & X86_CR4_PAE)) {
1813                         hw_cr4 &= ~X86_CR4_PAE;
1814                 }
1815         }
1816
1817         vmcs_writel(CR4_READ_SHADOW, cr4);
1818         vmcs_writel(GUEST_CR4, hw_cr4);
1819 }
1820
1821 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1822 {
1823         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1824
1825         return vmcs_readl(sf->base);
1826 }
1827
1828 static void vmx_get_segment(struct kvm_vcpu *vcpu,
1829                             struct kvm_segment *var, int seg)
1830 {
1831         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1832         u32 ar;
1833
1834         var->base = vmcs_readl(sf->base);
1835         var->limit = vmcs_read32(sf->limit);
1836         var->selector = vmcs_read16(sf->selector);
1837         ar = vmcs_read32(sf->ar_bytes);
1838         if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
1839                 ar = 0;
1840         var->type = ar & 15;
1841         var->s = (ar >> 4) & 1;
1842         var->dpl = (ar >> 5) & 3;
1843         var->present = (ar >> 7) & 1;
1844         var->avl = (ar >> 12) & 1;
1845         var->l = (ar >> 13) & 1;
1846         var->db = (ar >> 14) & 1;
1847         var->g = (ar >> 15) & 1;
1848         var->unusable = (ar >> 16) & 1;
1849 }
1850
1851 static int vmx_get_cpl(struct kvm_vcpu *vcpu)
1852 {
1853         if (!kvm_read_cr0_bits(vcpu, X86_CR0_PE)) /* if real mode */
1854                 return 0;
1855
1856         if (vmx_get_rflags(vcpu) & X86_EFLAGS_VM) /* if virtual 8086 */
1857                 return 3;
1858
1859         return vmcs_read16(GUEST_CS_SELECTOR) & 3;
1860 }
1861
1862 static u32 vmx_segment_access_rights(struct kvm_segment *var)
1863 {
1864         u32 ar;
1865
1866         if (var->unusable)
1867                 ar = 1 << 16;
1868         else {
1869                 ar = var->type & 15;
1870                 ar |= (var->s & 1) << 4;
1871                 ar |= (var->dpl & 3) << 5;
1872                 ar |= (var->present & 1) << 7;
1873                 ar |= (var->avl & 1) << 12;
1874                 ar |= (var->l & 1) << 13;
1875                 ar |= (var->db & 1) << 14;
1876                 ar |= (var->g & 1) << 15;
1877         }
1878         if (ar == 0) /* a 0 value means unusable */
1879                 ar = AR_UNUSABLE_MASK;
1880
1881         return ar;
1882 }
1883
1884 static void vmx_set_segment(struct kvm_vcpu *vcpu,
1885                             struct kvm_segment *var, int seg)
1886 {
1887         struct vcpu_vmx *vmx = to_vmx(vcpu);
1888         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1889         u32 ar;
1890
1891         if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
1892                 vmx->rmode.tr.selector = var->selector;
1893                 vmx->rmode.tr.base = var->base;
1894                 vmx->rmode.tr.limit = var->limit;
1895                 vmx->rmode.tr.ar = vmx_segment_access_rights(var);
1896                 return;
1897         }
1898         vmcs_writel(sf->base, var->base);
1899         vmcs_write32(sf->limit, var->limit);
1900         vmcs_write16(sf->selector, var->selector);
1901         if (vmx->rmode.vm86_active && var->s) {
1902                 /*
1903                  * Hack real-mode segments into vm86 compatibility.
1904                  */
1905                 if (var->base == 0xffff0000 && var->selector == 0xf000)
1906                         vmcs_writel(sf->base, 0xf0000);
1907                 ar = 0xf3;
1908         } else
1909                 ar = vmx_segment_access_rights(var);
1910
1911         /*
1912          *   Fix the "Accessed" bit in AR field of segment registers for older
1913          * qemu binaries.
1914          *   IA32 arch specifies that at the time of processor reset the
1915          * "Accessed" bit in the AR field of segment registers is 1. And qemu
1916          * is setting it to 0 in the usedland code. This causes invalid guest
1917          * state vmexit when "unrestricted guest" mode is turned on.
1918          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
1919          * tree. Newer qemu binaries with that qemu fix would not need this
1920          * kvm hack.
1921          */
1922         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
1923                 ar |= 0x1; /* Accessed */
1924
1925         vmcs_write32(sf->ar_bytes, ar);
1926 }
1927
1928 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1929 {
1930         u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
1931
1932         *db = (ar >> 14) & 1;
1933         *l = (ar >> 13) & 1;
1934 }
1935
1936 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1937 {
1938         dt->limit = vmcs_read32(GUEST_IDTR_LIMIT);
1939         dt->base = vmcs_readl(GUEST_IDTR_BASE);
1940 }
1941
1942 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1943 {
1944         vmcs_write32(GUEST_IDTR_LIMIT, dt->limit);
1945         vmcs_writel(GUEST_IDTR_BASE, dt->base);
1946 }
1947
1948 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1949 {
1950         dt->limit = vmcs_read32(GUEST_GDTR_LIMIT);
1951         dt->base = vmcs_readl(GUEST_GDTR_BASE);
1952 }
1953
1954 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1955 {
1956         vmcs_write32(GUEST_GDTR_LIMIT, dt->limit);
1957         vmcs_writel(GUEST_GDTR_BASE, dt->base);
1958 }
1959
1960 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
1961 {
1962         struct kvm_segment var;
1963         u32 ar;
1964
1965         vmx_get_segment(vcpu, &var, seg);
1966         ar = vmx_segment_access_rights(&var);
1967
1968         if (var.base != (var.selector << 4))
1969                 return false;
1970         if (var.limit != 0xffff)
1971                 return false;
1972         if (ar != 0xf3)
1973                 return false;
1974
1975         return true;
1976 }
1977
1978 static bool code_segment_valid(struct kvm_vcpu *vcpu)
1979 {
1980         struct kvm_segment cs;
1981         unsigned int cs_rpl;
1982
1983         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
1984         cs_rpl = cs.selector & SELECTOR_RPL_MASK;
1985
1986         if (cs.unusable)
1987                 return false;
1988         if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
1989                 return false;
1990         if (!cs.s)
1991                 return false;
1992         if (cs.type & AR_TYPE_WRITEABLE_MASK) {
1993                 if (cs.dpl > cs_rpl)
1994                         return false;
1995         } else {
1996                 if (cs.dpl != cs_rpl)
1997                         return false;
1998         }
1999         if (!cs.present)
2000                 return false;
2001
2002         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
2003         return true;
2004 }
2005
2006 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
2007 {
2008         struct kvm_segment ss;
2009         unsigned int ss_rpl;
2010
2011         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
2012         ss_rpl = ss.selector & SELECTOR_RPL_MASK;
2013
2014         if (ss.unusable)
2015                 return true;
2016         if (ss.type != 3 && ss.type != 7)
2017                 return false;
2018         if (!ss.s)
2019                 return false;
2020         if (ss.dpl != ss_rpl) /* DPL != RPL */
2021                 return false;
2022         if (!ss.present)
2023                 return false;
2024
2025         return true;
2026 }
2027
2028 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
2029 {
2030         struct kvm_segment var;
2031         unsigned int rpl;
2032
2033         vmx_get_segment(vcpu, &var, seg);
2034         rpl = var.selector & SELECTOR_RPL_MASK;
2035
2036         if (var.unusable)
2037                 return true;
2038         if (!var.s)
2039                 return false;
2040         if (!var.present)
2041                 return false;
2042         if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
2043                 if (var.dpl < rpl) /* DPL < RPL */
2044                         return false;
2045         }
2046
2047         /* TODO: Add other members to kvm_segment_field to allow checking for other access
2048          * rights flags
2049          */
2050         return true;
2051 }
2052
2053 static bool tr_valid(struct kvm_vcpu *vcpu)
2054 {
2055         struct kvm_segment tr;
2056
2057         vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
2058
2059         if (tr.unusable)
2060                 return false;
2061         if (tr.selector & SELECTOR_TI_MASK)     /* TI = 1 */
2062                 return false;
2063         if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
2064                 return false;
2065         if (!tr.present)
2066                 return false;
2067
2068         return true;
2069 }
2070
2071 static bool ldtr_valid(struct kvm_vcpu *vcpu)
2072 {
2073         struct kvm_segment ldtr;
2074
2075         vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
2076
2077         if (ldtr.unusable)
2078                 return true;
2079         if (ldtr.selector & SELECTOR_TI_MASK)   /* TI = 1 */
2080                 return false;
2081         if (ldtr.type != 2)
2082                 return false;
2083         if (!ldtr.present)
2084                 return false;
2085
2086         return true;
2087 }
2088
2089 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
2090 {
2091         struct kvm_segment cs, ss;
2092
2093         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
2094         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
2095
2096         return ((cs.selector & SELECTOR_RPL_MASK) ==
2097                  (ss.selector & SELECTOR_RPL_MASK));
2098 }
2099
2100 /*
2101  * Check if guest state is valid. Returns true if valid, false if
2102  * not.
2103  * We assume that registers are always usable
2104  */
2105 static bool guest_state_valid(struct kvm_vcpu *vcpu)
2106 {
2107         /* real mode guest state checks */
2108         if (!kvm_read_cr0_bits(vcpu, X86_CR0_PE)) {
2109                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
2110                         return false;
2111                 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
2112                         return false;
2113                 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
2114                         return false;
2115                 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
2116                         return false;
2117                 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
2118                         return false;
2119                 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
2120                         return false;
2121         } else {
2122         /* protected mode guest state checks */
2123                 if (!cs_ss_rpl_check(vcpu))
2124                         return false;
2125                 if (!code_segment_valid(vcpu))
2126                         return false;
2127                 if (!stack_segment_valid(vcpu))
2128                         return false;
2129                 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
2130                         return false;
2131                 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
2132                         return false;
2133                 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
2134                         return false;
2135                 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
2136                         return false;
2137                 if (!tr_valid(vcpu))
2138                         return false;
2139                 if (!ldtr_valid(vcpu))
2140                         return false;
2141         }
2142         /* TODO:
2143          * - Add checks on RIP
2144          * - Add checks on RFLAGS
2145          */
2146
2147         return true;
2148 }
2149
2150 static int init_rmode_tss(struct kvm *kvm)
2151 {
2152         gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
2153         u16 data = 0;
2154         int ret = 0;
2155         int r;
2156
2157         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2158         if (r < 0)
2159                 goto out;
2160         data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
2161         r = kvm_write_guest_page(kvm, fn++, &data,
2162                         TSS_IOPB_BASE_OFFSET, sizeof(u16));
2163         if (r < 0)
2164                 goto out;
2165         r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
2166         if (r < 0)
2167                 goto out;
2168         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2169         if (r < 0)
2170                 goto out;
2171         data = ~0;
2172         r = kvm_write_guest_page(kvm, fn, &data,
2173                                  RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
2174                                  sizeof(u8));
2175         if (r < 0)
2176                 goto out;
2177
2178         ret = 1;
2179 out:
2180         return ret;
2181 }
2182
2183 static int init_rmode_identity_map(struct kvm *kvm)
2184 {
2185         int i, r, ret;
2186         pfn_t identity_map_pfn;
2187         u32 tmp;
2188
2189         if (!enable_ept)
2190                 return 1;
2191         if (unlikely(!kvm->arch.ept_identity_pagetable)) {
2192                 printk(KERN_ERR "EPT: identity-mapping pagetable "
2193                         "haven't been allocated!\n");
2194                 return 0;
2195         }
2196         if (likely(kvm->arch.ept_identity_pagetable_done))
2197                 return 1;
2198         ret = 0;
2199         identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
2200         r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
2201         if (r < 0)
2202                 goto out;
2203         /* Set up identity-mapping pagetable for EPT in real mode */
2204         for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
2205                 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
2206                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
2207                 r = kvm_write_guest_page(kvm, identity_map_pfn,
2208                                 &tmp, i * sizeof(tmp), sizeof(tmp));
2209                 if (r < 0)
2210                         goto out;
2211         }
2212         kvm->arch.ept_identity_pagetable_done = true;
2213         ret = 1;
2214 out:
2215         return ret;
2216 }
2217
2218 static void seg_setup(int seg)
2219 {
2220         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2221         unsigned int ar;
2222
2223         vmcs_write16(sf->selector, 0);
2224         vmcs_writel(sf->base, 0);
2225         vmcs_write32(sf->limit, 0xffff);
2226         if (enable_unrestricted_guest) {
2227                 ar = 0x93;
2228                 if (seg == VCPU_SREG_CS)
2229                         ar |= 0x08; /* code segment */
2230         } else
2231                 ar = 0xf3;
2232
2233         vmcs_write32(sf->ar_bytes, ar);
2234 }
2235
2236 static int alloc_apic_access_page(struct kvm *kvm)
2237 {
2238         struct kvm_userspace_memory_region kvm_userspace_mem;
2239         int r = 0;
2240
2241         mutex_lock(&kvm->slots_lock);
2242         if (kvm->arch.apic_access_page)
2243                 goto out;
2244         kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
2245         kvm_userspace_mem.flags = 0;
2246         kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
2247         kvm_userspace_mem.memory_size = PAGE_SIZE;
2248         r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2249         if (r)
2250                 goto out;
2251
2252         kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
2253 out:
2254         mutex_unlock(&kvm->slots_lock);
2255         return r;
2256 }
2257
2258 static int alloc_identity_pagetable(struct kvm *kvm)
2259 {
2260         struct kvm_userspace_memory_region kvm_userspace_mem;
2261         int r = 0;
2262
2263         mutex_lock(&kvm->slots_lock);
2264         if (kvm->arch.ept_identity_pagetable)
2265                 goto out;
2266         kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
2267         kvm_userspace_mem.flags = 0;
2268         kvm_userspace_mem.guest_phys_addr =
2269                 kvm->arch.ept_identity_map_addr;
2270         kvm_userspace_mem.memory_size = PAGE_SIZE;
2271         r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2272         if (r)
2273                 goto out;
2274
2275         kvm->arch.ept_identity_pagetable = gfn_to_page(kvm,
2276                         kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
2277 out:
2278         mutex_unlock(&kvm->slots_lock);
2279         return r;
2280 }
2281
2282 static void allocate_vpid(struct vcpu_vmx *vmx)
2283 {
2284         int vpid;
2285
2286         vmx->vpid = 0;
2287         if (!enable_vpid)
2288                 return;
2289         spin_lock(&vmx_vpid_lock);
2290         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
2291         if (vpid < VMX_NR_VPIDS) {
2292                 vmx->vpid = vpid;
2293                 __set_bit(vpid, vmx_vpid_bitmap);
2294         }
2295         spin_unlock(&vmx_vpid_lock);
2296 }
2297
2298 static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
2299 {
2300         int f = sizeof(unsigned long);
2301
2302         if (!cpu_has_vmx_msr_bitmap())
2303                 return;
2304
2305         /*
2306          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
2307          * have the write-low and read-high bitmap offsets the wrong way round.
2308          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
2309          */
2310         if (msr <= 0x1fff) {
2311                 __clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
2312                 __clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
2313         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2314                 msr &= 0x1fff;
2315                 __clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
2316                 __clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
2317         }
2318 }
2319
2320 static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
2321 {
2322         if (!longmode_only)
2323                 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
2324         __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
2325 }
2326
2327 /*
2328  * Sets up the vmcs for emulated real mode.
2329  */
2330 static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
2331 {
2332         u32 host_sysenter_cs, msr_low, msr_high;
2333         u32 junk;
2334         u64 host_pat, tsc_this, tsc_base;
2335         unsigned long a;
2336         struct descriptor_table dt;
2337         int i;
2338         unsigned long kvm_vmx_return;
2339         u32 exec_control;
2340
2341         /* I/O */
2342         vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
2343         vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
2344
2345         if (cpu_has_vmx_msr_bitmap())
2346                 vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
2347
2348         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
2349
2350         /* Control */
2351         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
2352                 vmcs_config.pin_based_exec_ctrl);
2353
2354         exec_control = vmcs_config.cpu_based_exec_ctrl;
2355         if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
2356                 exec_control &= ~CPU_BASED_TPR_SHADOW;
2357 #ifdef CONFIG_X86_64
2358                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
2359                                 CPU_BASED_CR8_LOAD_EXITING;
2360 #endif
2361         }
2362         if (!enable_ept)
2363                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
2364                                 CPU_BASED_CR3_LOAD_EXITING  |
2365                                 CPU_BASED_INVLPG_EXITING;
2366         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
2367
2368         if (cpu_has_secondary_exec_ctrls()) {
2369                 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
2370                 if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2371                         exec_control &=
2372                                 ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
2373                 if (vmx->vpid == 0)
2374                         exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
2375                 if (!enable_ept) {
2376                         exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
2377                         enable_unrestricted_guest = 0;
2378                 }
2379                 if (!enable_unrestricted_guest)
2380                         exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
2381                 if (!ple_gap)
2382                         exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
2383                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
2384         }
2385
2386         if (ple_gap) {
2387                 vmcs_write32(PLE_GAP, ple_gap);
2388                 vmcs_write32(PLE_WINDOW, ple_window);
2389         }
2390
2391         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf);
2392         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf);
2393         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
2394
2395         vmcs_writel(HOST_CR0, read_cr0());  /* 22.2.3 */
2396         vmcs_writel(HOST_CR4, read_cr4());  /* 22.2.3, 22.2.5 */
2397         vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */
2398
2399         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
2400         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
2401         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
2402         vmcs_write16(HOST_FS_SELECTOR, kvm_read_fs());    /* 22.2.4 */
2403         vmcs_write16(HOST_GS_SELECTOR, kvm_read_gs());    /* 22.2.4 */
2404         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
2405 #ifdef CONFIG_X86_64
2406         rdmsrl(MSR_FS_BASE, a);
2407         vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
2408         rdmsrl(MSR_GS_BASE, a);
2409         vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
2410 #else
2411         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
2412         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
2413 #endif
2414
2415         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
2416
2417         kvm_get_idt(&dt);
2418         vmcs_writel(HOST_IDTR_BASE, dt.base);   /* 22.2.4 */
2419
2420         asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
2421         vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
2422         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
2423         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2424         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
2425
2426         rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
2427         vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
2428         rdmsrl(MSR_IA32_SYSENTER_ESP, a);
2429         vmcs_writel(HOST_IA32_SYSENTER_ESP, a);   /* 22.2.3 */
2430         rdmsrl(MSR_IA32_SYSENTER_EIP, a);
2431         vmcs_writel(HOST_IA32_SYSENTER_EIP, a);   /* 22.2.3 */
2432
2433         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
2434                 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2435                 host_pat = msr_low | ((u64) msr_high << 32);
2436                 vmcs_write64(HOST_IA32_PAT, host_pat);
2437         }
2438         if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2439                 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2440                 host_pat = msr_low | ((u64) msr_high << 32);
2441                 /* Write the default value follow host pat */
2442                 vmcs_write64(GUEST_IA32_PAT, host_pat);
2443                 /* Keep arch.pat sync with GUEST_IA32_PAT */
2444                 vmx->vcpu.arch.pat = host_pat;
2445         }
2446
2447         for (i = 0; i < NR_VMX_MSR; ++i) {
2448                 u32 index = vmx_msr_index[i];
2449                 u32 data_low, data_high;
2450                 int j = vmx->nmsrs;
2451
2452                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
2453                         continue;
2454                 if (wrmsr_safe(index, data_low, data_high) < 0)
2455                         continue;
2456                 vmx->guest_msrs[j].index = i;
2457                 vmx->guest_msrs[j].data = 0;
2458                 vmx->guest_msrs[j].mask = -1ull;
2459                 ++vmx->nmsrs;
2460         }
2461
2462         vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
2463
2464         /* 22.2.1, 20.8.1 */
2465         vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);
2466
2467         vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
2468         vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
2469         if (enable_ept)
2470                 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
2471         vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
2472
2473         tsc_base = vmx->vcpu.kvm->arch.vm_init_tsc;
2474         rdtscll(tsc_this);
2475         if (tsc_this < vmx->vcpu.kvm->arch.vm_init_tsc)
2476                 tsc_base = tsc_this;
2477
2478         guest_write_tsc(0, tsc_base);
2479
2480         return 0;
2481 }
2482
2483 static int init_rmode(struct kvm *kvm)
2484 {
2485         if (!init_rmode_tss(kvm))
2486                 return 0;
2487         if (!init_rmode_identity_map(kvm))
2488                 return 0;
2489         return 1;
2490 }
2491
2492 static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
2493 {
2494         struct vcpu_vmx *vmx = to_vmx(vcpu);
2495         u64 msr;
2496         int ret, idx;
2497
2498         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));
2499         idx = srcu_read_lock(&vcpu->kvm->srcu);
2500         if (!init_rmode(vmx->vcpu.kvm)) {
2501                 ret = -ENOMEM;
2502                 goto out;
2503         }
2504
2505         vmx->rmode.vm86_active = 0;
2506
2507         vmx->soft_vnmi_blocked = 0;
2508
2509         vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
2510         kvm_set_cr8(&vmx->vcpu, 0);
2511         msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
2512         if (kvm_vcpu_is_bsp(&vmx->vcpu))
2513                 msr |= MSR_IA32_APICBASE_BSP;
2514         kvm_set_apic_base(&vmx->vcpu, msr);
2515
2516         fx_init(&vmx->vcpu);
2517
2518         seg_setup(VCPU_SREG_CS);
2519         /*
2520          * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
2521          * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
2522          */
2523         if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
2524                 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
2525                 vmcs_writel(GUEST_CS_BASE, 0x000f0000);
2526         } else {
2527                 vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
2528                 vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
2529         }
2530
2531         seg_setup(VCPU_SREG_DS);
2532         seg_setup(VCPU_SREG_ES);
2533         seg_setup(VCPU_SREG_FS);
2534         seg_setup(VCPU_SREG_GS);
2535         seg_setup(VCPU_SREG_SS);
2536
2537         vmcs_write16(GUEST_TR_SELECTOR, 0);
2538         vmcs_writel(GUEST_TR_BASE, 0);
2539         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
2540         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2541
2542         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
2543         vmcs_writel(GUEST_LDTR_BASE, 0);
2544         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
2545         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
2546
2547         vmcs_write32(GUEST_SYSENTER_CS, 0);
2548         vmcs_writel(GUEST_SYSENTER_ESP, 0);
2549         vmcs_writel(GUEST_SYSENTER_EIP, 0);
2550
2551         vmcs_writel(GUEST_RFLAGS, 0x02);
2552         if (kvm_vcpu_is_bsp(&vmx->vcpu))
2553                 kvm_rip_write(vcpu, 0xfff0);
2554         else
2555                 kvm_rip_write(vcpu, 0);
2556         kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
2557
2558         vmcs_writel(GUEST_DR7, 0x400);
2559
2560         vmcs_writel(GUEST_GDTR_BASE, 0);
2561         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
2562
2563         vmcs_writel(GUEST_IDTR_BASE, 0);
2564         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
2565
2566         vmcs_write32(GUEST_ACTIVITY_STATE, 0);
2567         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
2568         vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
2569
2570         /* Special registers */
2571         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
2572
2573         setup_msrs(vmx);
2574
2575         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
2576
2577         if (cpu_has_vmx_tpr_shadow()) {
2578                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
2579                 if (vm_need_tpr_shadow(vmx->vcpu.kvm))
2580                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
2581                                 page_to_phys(vmx->vcpu.arch.apic->regs_page));
2582                 vmcs_write32(TPR_THRESHOLD, 0);
2583         }
2584
2585         if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2586                 vmcs_write64(APIC_ACCESS_ADDR,
2587                              page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
2588
2589         if (vmx->vpid != 0)
2590                 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2591
2592         vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
2593         vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
2594         vmx_set_cr4(&vmx->vcpu, 0);
2595         vmx_set_efer(&vmx->vcpu, 0);
2596         vmx_fpu_activate(&vmx->vcpu);
2597         update_exception_bitmap(&vmx->vcpu);
2598
2599         vpid_sync_vcpu_all(vmx);
2600
2601         ret = 0;
2602
2603         /* HACK: Don't enable emulation on guest boot/reset */
2604         vmx->emulation_required = 0;
2605
2606 out:
2607         srcu_read_unlock(&vcpu->kvm->srcu, idx);
2608         return ret;
2609 }
2610
2611 static void enable_irq_window(struct kvm_vcpu *vcpu)
2612 {
2613         u32 cpu_based_vm_exec_control;
2614
2615         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2616         cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
2617         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2618 }
2619
2620 static void enable_nmi_window(struct kvm_vcpu *vcpu)
2621 {
2622         u32 cpu_based_vm_exec_control;
2623
2624         if (!cpu_has_virtual_nmis()) {
2625                 enable_irq_window(vcpu);
2626                 return;
2627         }
2628
2629         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2630         cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
2631         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2632 }
2633
2634 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
2635 {
2636         struct vcpu_vmx *vmx = to_vmx(vcpu);
2637         uint32_t intr;
2638         int irq = vcpu->arch.interrupt.nr;
2639
2640         trace_kvm_inj_virq(irq);
2641
2642         ++vcpu->stat.irq_injections;
2643         if (vmx->rmode.vm86_active) {
2644                 vmx->rmode.irq.pending = true;
2645                 vmx->rmode.irq.vector = irq;
2646                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
2647                 if (vcpu->arch.interrupt.soft)
2648                         vmx->rmode.irq.rip +=
2649                                 vmx->vcpu.arch.event_exit_inst_len;
2650                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2651                              irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK);
2652                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
2653                 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
2654                 return;
2655         }
2656         intr = irq | INTR_INFO_VALID_MASK;
2657         if (vcpu->arch.interrupt.soft) {
2658                 intr |= INTR_TYPE_SOFT_INTR;
2659                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2660                              vmx->vcpu.arch.event_exit_inst_len);
2661         } else
2662                 intr |= INTR_TYPE_EXT_INTR;
2663         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
2664 }
2665
2666 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
2667 {
2668         struct vcpu_vmx *vmx = to_vmx(vcpu);
2669
2670         if (!cpu_has_virtual_nmis()) {
2671                 /*
2672                  * Tracking the NMI-blocked state in software is built upon
2673                  * finding the next open IRQ window. This, in turn, depends on
2674                  * well-behaving guests: They have to keep IRQs disabled at
2675                  * least as long as the NMI handler runs. Otherwise we may
2676                  * cause NMI nesting, maybe breaking the guest. But as this is
2677                  * highly unlikely, we can live with the residual risk.
2678                  */
2679                 vmx->soft_vnmi_blocked = 1;
2680                 vmx->vnmi_blocked_time = 0;
2681         }
2682
2683         ++vcpu->stat.nmi_injections;
2684         if (vmx->rmode.vm86_active) {
2685                 vmx->rmode.irq.pending = true;
2686                 vmx->rmode.irq.vector = NMI_VECTOR;
2687                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
2688                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2689                              NMI_VECTOR | INTR_TYPE_SOFT_INTR |
2690                              INTR_INFO_VALID_MASK);
2691                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
2692                 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
2693                 return;
2694         }
2695         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2696                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
2697 }
2698
2699 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
2700 {
2701         if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
2702                 return 0;
2703
2704         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2705                         (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS |
2706                                 GUEST_INTR_STATE_NMI));
2707 }
2708
2709 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
2710 {
2711         if (!cpu_has_virtual_nmis())
2712                 return to_vmx(vcpu)->soft_vnmi_blocked;
2713         else
2714                 return !!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2715                           GUEST_INTR_STATE_NMI);
2716 }
2717
2718 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
2719 {
2720         struct vcpu_vmx *vmx = to_vmx(vcpu);
2721
2722         if (!cpu_has_virtual_nmis()) {
2723                 if (vmx->soft_vnmi_blocked != masked) {
2724                         vmx->soft_vnmi_blocked = masked;
2725                         vmx->vnmi_blocked_time = 0;
2726                 }
2727         } else {
2728                 if (masked)
2729                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
2730                                       GUEST_INTR_STATE_NMI);
2731                 else
2732                         vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
2733                                         GUEST_INTR_STATE_NMI);
2734         }
2735 }
2736
2737 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
2738 {
2739         return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
2740                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2741                         (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
2742 }
2743
2744 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
2745 {
2746         int ret;
2747         struct kvm_userspace_memory_region tss_mem = {
2748                 .slot = TSS_PRIVATE_MEMSLOT,
2749                 .guest_phys_addr = addr,
2750                 .memory_size = PAGE_SIZE * 3,
2751                 .flags = 0,
2752         };
2753
2754         ret = kvm_set_memory_region(kvm, &tss_mem, 0);
2755         if (ret)
2756                 return ret;
2757         kvm->arch.tss_addr = addr;
2758         return 0;
2759 }
2760
2761 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
2762                                   int vec, u32 err_code)
2763 {
2764         /*
2765          * Instruction with address size override prefix opcode 0x67
2766          * Cause the #SS fault with 0 error code in VM86 mode.
2767          */
2768         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
2769                 if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE)
2770                         return 1;
2771         /*
2772          * Forward all other exceptions that are valid in real mode.
2773          * FIXME: Breaks guest debugging in real mode, needs to be fixed with
2774          *        the required debugging infrastructure rework.
2775          */
2776         switch (vec) {
2777         case DB_VECTOR:
2778                 if (vcpu->guest_debug &
2779                     (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
2780                         return 0;
2781                 kvm_queue_exception(vcpu, vec);
2782                 return 1;
2783         case BP_VECTOR:
2784                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2785                         return 0;
2786                 /* fall through */
2787         case DE_VECTOR:
2788         case OF_VECTOR:
2789         case BR_VECTOR:
2790         case UD_VECTOR:
2791         case DF_VECTOR:
2792         case SS_VECTOR:
2793         case GP_VECTOR:
2794         case MF_VECTOR:
2795                 kvm_queue_exception(vcpu, vec);
2796                 return 1;
2797         }
2798         return 0;
2799 }
2800
2801 /*
2802  * Trigger machine check on the host. We assume all the MSRs are already set up
2803  * by the CPU and that we still run on the same CPU as the MCE occurred on.
2804  * We pass a fake environment to the machine check handler because we want
2805  * the guest to be always treated like user space, no matter what context
2806  * it used internally.
2807  */
2808 static void kvm_machine_check(void)
2809 {
2810 #if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
2811         struct pt_regs regs = {
2812                 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
2813                 .flags = X86_EFLAGS_IF,
2814         };
2815
2816         do_machine_check(&regs, 0);
2817 #endif
2818 }
2819
2820 static int handle_machine_check(struct kvm_vcpu *vcpu)
2821 {
2822         /* already handled by vcpu_run */
2823         return 1;
2824 }
2825
2826 static int handle_exception(struct kvm_vcpu *vcpu)
2827 {
2828         struct vcpu_vmx *vmx = to_vmx(vcpu);
2829         struct kvm_run *kvm_run = vcpu->run;
2830         u32 intr_info, ex_no, error_code;
2831         unsigned long cr2, rip, dr6;
2832         u32 vect_info;
2833         enum emulation_result er;
2834
2835         vect_info = vmx->idt_vectoring_info;
2836         intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
2837
2838         if (is_machine_check(intr_info))
2839                 return handle_machine_check(vcpu);
2840
2841         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
2842             !is_page_fault(intr_info)) {
2843                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2844                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
2845                 vcpu->run->internal.ndata = 2;
2846                 vcpu->run->internal.data[0] = vect_info;
2847                 vcpu->run->internal.data[1] = intr_info;
2848                 return 0;
2849         }
2850
2851         if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
2852                 return 1;  /* already handled by vmx_vcpu_run() */
2853
2854         if (is_no_device(intr_info)) {
2855                 vmx_fpu_activate(vcpu);
2856                 return 1;
2857         }
2858
2859         if (is_invalid_opcode(intr_info)) {
2860                 er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD);
2861                 if (er != EMULATE_DONE)
2862                         kvm_queue_exception(vcpu, UD_VECTOR);
2863                 return 1;
2864         }
2865
2866         error_code = 0;
2867         rip = kvm_rip_read(vcpu);
2868         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
2869                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
2870         if (is_page_fault(intr_info)) {
2871                 /* EPT won't cause page fault directly */
2872                 if (enable_ept)
2873                         BUG();
2874                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
2875                 trace_kvm_page_fault(cr2, error_code);
2876
2877                 if (kvm_event_needs_reinjection(vcpu))
2878                         kvm_mmu_unprotect_page_virt(vcpu, cr2);
2879                 return kvm_mmu_page_fault(vcpu, cr2, error_code);
2880         }
2881
2882         if (vmx->rmode.vm86_active &&
2883             handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
2884                                                                 error_code)) {
2885                 if (vcpu->arch.halt_request) {
2886                         vcpu->arch.halt_request = 0;
2887                         return kvm_emulate_halt(vcpu);
2888                 }
2889                 return 1;
2890         }
2891
2892         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
2893         switch (ex_no) {
2894         case DB_VECTOR:
2895                 dr6 = vmcs_readl(EXIT_QUALIFICATION);
2896                 if (!(vcpu->guest_debug &
2897                       (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
2898                         vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
2899                         kvm_queue_exception(vcpu, DB_VECTOR);
2900                         return 1;
2901                 }
2902                 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
2903                 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
2904                 /* fall through */
2905         case BP_VECTOR:
2906                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2907                 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
2908                 kvm_run->debug.arch.exception = ex_no;
2909                 break;
2910         default:
2911                 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
2912                 kvm_run->ex.exception = ex_no;
2913                 kvm_run->ex.error_code = error_code;
2914                 break;
2915         }
2916         return 0;
2917 }
2918
2919 static int handle_external_interrupt(struct kvm_vcpu *vcpu)
2920 {
2921         ++vcpu->stat.irq_exits;
2922         return 1;
2923 }
2924
2925 static int handle_triple_fault(struct kvm_vcpu *vcpu)
2926 {
2927         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
2928         return 0;
2929 }
2930
2931 static int handle_io(struct kvm_vcpu *vcpu)
2932 {
2933         unsigned long exit_qualification;
2934         int size, in, string;
2935         unsigned port;
2936
2937         ++vcpu->stat.io_exits;
2938         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
2939         string = (exit_qualification & 16) != 0;
2940
2941         if (string) {
2942                 if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DO_MMIO)
2943                         return 0;
2944                 return 1;
2945         }
2946
2947         size = (exit_qualification & 7) + 1;
2948         in = (exit_qualification & 8) != 0;
2949         port = exit_qualification >> 16;
2950
2951         skip_emulated_instruction(vcpu);
2952         return kvm_emulate_pio(vcpu, in, size, port);
2953 }
2954
2955 static void
2956 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
2957 {
2958         /*
2959          * Patch in the VMCALL instruction:
2960          */
2961         hypercall[0] = 0x0f;
2962         hypercall[1] = 0x01;
2963         hypercall[2] = 0xc1;
2964 }
2965
2966 static int handle_cr(struct kvm_vcpu *vcpu)
2967 {
2968         unsigned long exit_qualification, val;
2969         int cr;
2970         int reg;
2971
2972         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
2973         cr = exit_qualification & 15;
2974         reg = (exit_qualification >> 8) & 15;
2975         switch ((exit_qualification >> 4) & 3) {
2976         case 0: /* mov to cr */
2977                 val = kvm_register_read(vcpu, reg);
2978                 trace_kvm_cr_write(cr, val);
2979                 switch (cr) {
2980                 case 0:
2981                         kvm_set_cr0(vcpu, val);
2982                         skip_emulated_instruction(vcpu);
2983                         return 1;
2984                 case 3:
2985                         kvm_set_cr3(vcpu, val);
2986                         skip_emulated_instruction(vcpu);
2987                         return 1;
2988                 case 4:
2989                         kvm_set_cr4(vcpu, val);
2990                         skip_emulated_instruction(vcpu);
2991                         return 1;
2992                 case 8: {
2993                                 u8 cr8_prev = kvm_get_cr8(vcpu);
2994                                 u8 cr8 = kvm_register_read(vcpu, reg);
2995                                 kvm_set_cr8(vcpu, cr8);
2996                                 skip_emulated_instruction(vcpu);
2997                                 if (irqchip_in_kernel(vcpu->kvm))
2998                                         return 1;
2999                                 if (cr8_prev <= cr8)
3000                                         return 1;
3001                                 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
3002                                 return 0;
3003                         }
3004                 };
3005                 break;
3006         case 2: /* clts */
3007                 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
3008                 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
3009                 skip_emulated_instruction(vcpu);
3010                 return 1;
3011         case 1: /*mov from cr*/
3012                 switch (cr) {
3013                 case 3:
3014                         kvm_register_write(vcpu, reg, vcpu->arch.cr3);
3015                         trace_kvm_cr_read(cr, vcpu->arch.cr3);
3016                         skip_emulated_instruction(vcpu);
3017                         return 1;
3018                 case 8:
3019                         val = kvm_get_cr8(vcpu);
3020                         kvm_register_write(vcpu, reg, val);
3021                         trace_kvm_cr_read(cr, val);
3022                         skip_emulated_instruction(vcpu);
3023                         return 1;
3024                 }
3025                 break;
3026         case 3: /* lmsw */
3027                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
3028                 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
3029                 kvm_lmsw(vcpu, val);
3030
3031                 skip_emulated_instruction(vcpu);
3032                 return 1;
3033         default:
3034                 break;
3035         }
3036         vcpu->run->exit_reason = 0;
3037         pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
3038                (int)(exit_qualification >> 4) & 3, cr);
3039         return 0;
3040 }
3041
3042 static int check_dr_alias(struct kvm_vcpu *vcpu)
3043 {
3044         if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) {
3045                 kvm_queue_exception(vcpu, UD_VECTOR);
3046                 return -1;
3047         }
3048         return 0;
3049 }
3050
3051 static int handle_dr(struct kvm_vcpu *vcpu)
3052 {
3053         unsigned long exit_qualification;
3054         unsigned long val;
3055         int dr, reg;
3056
3057         /* Do not handle if the CPL > 0, will trigger GP on re-entry */
3058         if (!kvm_require_cpl(vcpu, 0))
3059                 return 1;
3060         dr = vmcs_readl(GUEST_DR7);
3061         if (dr & DR7_GD) {
3062                 /*
3063                  * As the vm-exit takes precedence over the debug trap, we
3064                  * need to emulate the latter, either for the host or the
3065                  * guest debugging itself.
3066                  */
3067                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
3068                         vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
3069                         vcpu->run->debug.arch.dr7 = dr;
3070                         vcpu->run->debug.arch.pc =
3071                                 vmcs_readl(GUEST_CS_BASE) +
3072                                 vmcs_readl(GUEST_RIP);
3073                         vcpu->run->debug.arch.exception = DB_VECTOR;
3074                         vcpu->run->exit_reason = KVM_EXIT_DEBUG;
3075                         return 0;
3076                 } else {
3077                         vcpu->arch.dr7 &= ~DR7_GD;
3078                         vcpu->arch.dr6 |= DR6_BD;
3079                         vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
3080                         kvm_queue_exception(vcpu, DB_VECTOR);
3081                         return 1;
3082                 }
3083         }
3084
3085         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3086         dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
3087         reg = DEBUG_REG_ACCESS_REG(exit_qualification);
3088         if (exit_qualification & TYPE_MOV_FROM_DR) {
3089                 switch (dr) {
3090                 case 0 ... 3:
3091                         val = vcpu->arch.db[dr];
3092                         break;
3093                 case 4:
3094                         if (check_dr_alias(vcpu) < 0)
3095                                 return 1;
3096                         /* fall through */
3097                 case 6:
3098                         val = vcpu->arch.dr6;
3099                         break;
3100                 case 5:
3101                         if (check_dr_alias(vcpu) < 0)
3102                                 return 1;
3103                         /* fall through */
3104                 default: /* 7 */
3105                         val = vcpu->arch.dr7;
3106                         break;
3107                 }
3108                 kvm_register_write(vcpu, reg, val);
3109         } else {
3110                 val = vcpu->arch.regs[reg];
3111                 switch (dr) {
3112                 case 0 ... 3:
3113                         vcpu->arch.db[dr] = val;
3114                         if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
3115                                 vcpu->arch.eff_db[dr] = val;
3116                         break;
3117                 case 4:
3118                         if (check_dr_alias(vcpu) < 0)
3119                                 return 1;
3120                         /* fall through */
3121                 case 6:
3122                         if (val & 0xffffffff00000000ULL) {
3123                                 kvm_inject_gp(vcpu, 0);
3124                                 return 1;
3125                         }
3126                         vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
3127                         break;
3128                 case 5:
3129                         if (check_dr_alias(vcpu) < 0)
3130                                 return 1;
3131                         /* fall through */
3132                 default: /* 7 */
3133                         if (val & 0xffffffff00000000ULL) {
3134                                 kvm_inject_gp(vcpu, 0);
3135                                 return 1;
3136                         }
3137                         vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
3138                         if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
3139                                 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
3140                                 vcpu->arch.switch_db_regs =
3141                                         (val & DR7_BP_EN_MASK);
3142                         }
3143                         break;
3144                 }
3145         }
3146         skip_emulated_instruction(vcpu);
3147         return 1;
3148 }
3149
3150 static int handle_cpuid(struct kvm_vcpu *vcpu)
3151 {
3152         kvm_emulate_cpuid(vcpu);
3153         return 1;
3154 }
3155
3156 static int handle_rdmsr(struct kvm_vcpu *vcpu)
3157 {
3158         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
3159         u64 data;
3160
3161         if (vmx_get_msr(vcpu, ecx, &data)) {
3162                 kvm_inject_gp(vcpu, 0);
3163                 return 1;
3164         }
3165
3166         trace_kvm_msr_read(ecx, data);
3167
3168         /* FIXME: handling of bits 32:63 of rax, rdx */
3169         vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
3170         vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
3171         skip_emulated_instruction(vcpu);
3172         return 1;
3173 }
3174
3175 static int handle_wrmsr(struct kvm_vcpu *vcpu)
3176 {
3177         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
3178         u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
3179                 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
3180
3181         trace_kvm_msr_write(ecx, data);
3182
3183         if (vmx_set_msr(vcpu, ecx, data) != 0) {
3184                 kvm_inject_gp(vcpu, 0);
3185                 return 1;
3186         }
3187
3188         skip_emulated_instruction(vcpu);
3189         return 1;
3190 }
3191
3192 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
3193 {
3194         return 1;
3195 }
3196
3197 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
3198 {
3199         u32 cpu_based_vm_exec_control;
3200
3201         /* clear pending irq */
3202         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3203         cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
3204         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
3205
3206         ++vcpu->stat.irq_window_exits;
3207
3208         /*
3209          * If the user space waits to inject interrupts, exit as soon as
3210          * possible
3211          */
3212         if (!irqchip_in_kernel(vcpu->kvm) &&
3213             vcpu->run->request_interrupt_window &&
3214             !kvm_cpu_has_interrupt(vcpu)) {
3215                 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
3216                 return 0;
3217         }
3218         return 1;
3219 }
3220
3221 static int handle_halt(struct kvm_vcpu *vcpu)
3222 {
3223         skip_emulated_instruction(vcpu);
3224         return kvm_emulate_halt(vcpu);
3225 }
3226
3227 static int handle_vmcall(struct kvm_vcpu *vcpu)
3228 {
3229         skip_emulated_instruction(vcpu);
3230         kvm_emulate_hypercall(vcpu);
3231         return 1;
3232 }
3233
3234 static int handle_vmx_insn(struct kvm_vcpu *vcpu)
3235 {
3236         kvm_queue_exception(vcpu, UD_VECTOR);
3237         return 1;
3238 }
3239
3240 static int handle_invlpg(struct kvm_vcpu *vcpu)
3241 {
3242         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3243
3244         kvm_mmu_invlpg(vcpu, exit_qualification);
3245         skip_emulated_instruction(vcpu);
3246         return 1;
3247 }
3248
3249 static int handle_wbinvd(struct kvm_vcpu *vcpu)
3250 {
3251         skip_emulated_instruction(vcpu);
3252         /* TODO: Add support for VT-d/pass-through device */
3253         return 1;
3254 }
3255
3256 static int handle_apic_access(struct kvm_vcpu *vcpu)
3257 {
3258         unsigned long exit_qualification;
3259         enum emulation_result er;
3260         unsigned long offset;
3261
3262         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3263         offset = exit_qualification & 0xffful;
3264
3265         er = emulate_instruction(vcpu, 0, 0, 0);
3266
3267         if (er !=  EMULATE_DONE) {
3268                 printk(KERN_ERR
3269                        "Fail to handle apic access vmexit! Offset is 0x%lx\n",
3270                        offset);
3271                 return -ENOEXEC;
3272         }
3273         return 1;
3274 }
3275
3276 static int handle_task_switch(struct kvm_vcpu *vcpu)
3277 {
3278         struct vcpu_vmx *vmx = to_vmx(vcpu);
3279         unsigned long exit_qualification;
3280         u16 tss_selector;
3281         int reason, type, idt_v;
3282
3283         idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
3284         type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
3285
3286         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3287
3288         reason = (u32)exit_qualification >> 30;
3289         if (reason == TASK_SWITCH_GATE && idt_v) {
3290                 switch (type) {
3291                 case INTR_TYPE_NMI_INTR:
3292                         vcpu->arch.nmi_injected = false;
3293                         if (cpu_has_virtual_nmis())
3294                                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3295                                               GUEST_INTR_STATE_NMI);
3296                         break;
3297                 case INTR_TYPE_EXT_INTR:
3298                 case INTR_TYPE_SOFT_INTR:
3299                         kvm_clear_interrupt_queue(vcpu);
3300                         break;
3301                 case INTR_TYPE_HARD_EXCEPTION:
3302                 case INTR_TYPE_SOFT_EXCEPTION:
3303                         kvm_clear_exception_queue(vcpu);
3304                         break;
3305                 default:
3306                         break;
3307                 }
3308         }
3309         tss_selector = exit_qualification;
3310
3311         if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
3312                        type != INTR_TYPE_EXT_INTR &&
3313                        type != INTR_TYPE_NMI_INTR))
3314                 skip_emulated_instruction(vcpu);
3315
3316         if (!kvm_task_switch(vcpu, tss_selector, reason))
3317                 return 0;
3318
3319         /* clear all local breakpoint enable flags */
3320         vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);
3321
3322         /*
3323          * TODO: What about debug traps on tss switch?
3324          *       Are we supposed to inject them and update dr6?
3325          */
3326
3327         return 1;
3328 }
3329
3330 static int handle_ept_violation(struct kvm_vcpu *vcpu)
3331 {
3332         unsigned long exit_qualification;
3333         gpa_t gpa;
3334         int gla_validity;
3335
3336         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3337
3338         if (exit_qualification & (1 << 6)) {
3339                 printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
3340                 return -EINVAL;
3341         }
3342
3343         gla_validity = (exit_qualification >> 7) & 0x3;
3344         if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
3345                 printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
3346                 printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
3347                         (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
3348                         vmcs_readl(GUEST_LINEAR_ADDRESS));
3349                 printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
3350                         (long unsigned int)exit_qualification);
3351                 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3352                 vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
3353                 return 0;
3354         }
3355
3356         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
3357         trace_kvm_page_fault(gpa, exit_qualification);
3358         return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0);
3359 }
3360
3361 static u64 ept_rsvd_mask(u64 spte, int level)
3362 {
3363         int i;
3364         u64 mask = 0;
3365
3366         for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
3367                 mask |= (1ULL << i);
3368
3369         if (level > 2)
3370                 /* bits 7:3 reserved */
3371                 mask |= 0xf8;
3372         else if (level == 2) {
3373                 if (spte & (1ULL << 7))
3374                         /* 2MB ref, bits 20:12 reserved */
3375                         mask |= 0x1ff000;
3376                 else
3377                         /* bits 6:3 reserved */
3378                         mask |= 0x78;
3379         }
3380
3381         return mask;
3382 }
3383
3384 static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
3385                                        int level)
3386 {
3387         printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);
3388
3389         /* 010b (write-only) */
3390         WARN_ON((spte & 0x7) == 0x2);
3391
3392         /* 110b (write/execute) */
3393         WARN_ON((spte & 0x7) == 0x6);
3394
3395         /* 100b (execute-only) and value not supported by logical processor */
3396         if (!cpu_has_vmx_ept_execute_only())
3397                 WARN_ON((spte & 0x7) == 0x4);
3398
3399         /* not 000b */
3400         if ((spte & 0x7)) {
3401                 u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);
3402
3403                 if (rsvd_bits != 0) {
3404                         printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
3405                                          __func__, rsvd_bits);
3406                         WARN_ON(1);
3407                 }
3408
3409                 if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
3410                         u64 ept_mem_type = (spte & 0x38) >> 3;
3411
3412                         if (ept_mem_type == 2 || ept_mem_type == 3 ||
3413                             ept_mem_type == 7) {
3414                                 printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
3415                                                 __func__, ept_mem_type);
3416                                 WARN_ON(1);
3417                         }
3418                 }
3419         }
3420 }
3421
3422 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
3423 {
3424         u64 sptes[4];
3425         int nr_sptes, i;
3426         gpa_t gpa;
3427
3428         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
3429
3430         printk(KERN_ERR "EPT: Misconfiguration.\n");
3431         printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);
3432
3433         nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);
3434
3435         for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
3436                 ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);
3437
3438         vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3439         vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
3440
3441         return 0;
3442 }
3443
3444 static int handle_nmi_window(struct kvm_vcpu *vcpu)
3445 {
3446         u32 cpu_based_vm_exec_control;
3447
3448         /* clear pending NMI */
3449         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3450         cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
3451         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
3452         ++vcpu->stat.nmi_window_exits;
3453
3454         return 1;
3455 }
3456
3457 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
3458 {
3459         struct vcpu_vmx *vmx = to_vmx(vcpu);
3460         enum emulation_result err = EMULATE_DONE;
3461         int ret = 1;
3462
3463         while (!guest_state_valid(vcpu)) {
3464                 err = emulate_instruction(vcpu, 0, 0, 0);
3465
3466                 if (err == EMULATE_DO_MMIO) {
3467                         ret = 0;
3468                         goto out;
3469                 }
3470
3471                 if (err != EMULATE_DONE) {
3472                         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3473                         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3474                         vcpu->run->internal.ndata = 0;
3475                         ret = 0;
3476                         goto out;
3477                 }
3478
3479                 if (signal_pending(current))
3480                         goto out;
3481                 if (need_resched())
3482                         schedule();
3483         }
3484
3485         vmx->emulation_required = 0;
3486 out:
3487         return ret;
3488 }
3489
3490 /*
3491  * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
3492  * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
3493  */
3494 static int handle_pause(struct kvm_vcpu *vcpu)
3495 {
3496         skip_emulated_instruction(vcpu);
3497         kvm_vcpu_on_spin(vcpu);
3498
3499         return 1;
3500 }
3501
3502 static int handle_invalid_op(struct kvm_vcpu *vcpu)
3503 {
3504         kvm_queue_exception(vcpu, UD_VECTOR);
3505         return 1;
3506 }
3507
3508 /*
3509  * The exit handlers return 1 if the exit was handled fully and guest execution
3510  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
3511  * to be done to userspace and return 0.
3512  */
3513 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3514         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
3515         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
3516         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
3517         [EXIT_REASON_NMI_WINDOW]              = handle_nmi_window,
3518         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
3519         [EXIT_REASON_CR_ACCESS]               = handle_cr,
3520         [EXIT_REASON_DR_ACCESS]               = handle_dr,
3521         [EXIT_REASON_CPUID]                   = handle_cpuid,
3522         [EXIT_REASON_MSR_READ]                = handle_rdmsr,
3523         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
3524         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
3525         [EXIT_REASON_HLT]                     = handle_halt,
3526         [EXIT_REASON_INVLPG]                  = handle_invlpg,
3527         [EXIT_REASON_VMCALL]                  = handle_vmcall,
3528         [EXIT_REASON_VMCLEAR]                 = handle_vmx_insn,
3529         [EXIT_REASON_VMLAUNCH]                = handle_vmx_insn,
3530         [EXIT_REASON_VMPTRLD]                 = handle_vmx_insn,
3531         [EXIT_REASON_VMPTRST]                 = handle_vmx_insn,
3532         [EXIT_REASON_VMREAD]                  = handle_vmx_insn,
3533         [EXIT_REASON_VMRESUME]                = handle_vmx_insn,
3534         [EXIT_REASON_VMWRITE]                 = handle_vmx_insn,
3535         [EXIT_REASON_VMOFF]                   = handle_vmx_insn,
3536         [EXIT_REASON_VMON]                    = handle_vmx_insn,
3537         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
3538         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
3539         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
3540         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
3541         [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
3542         [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
3543         [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
3544         [EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
3545         [EXIT_REASON_MWAIT_INSTRUCTION]       = handle_invalid_op,
3546         [EXIT_REASON_MONITOR_INSTRUCTION]     = handle_invalid_op,
3547 };
3548
3549 static const int kvm_vmx_max_exit_handlers =
3550         ARRAY_SIZE(kvm_vmx_exit_handlers);
3551
3552 /*
3553  * The guest has exited.  See if we can fix it or if we need userspace
3554  * assistance.
3555  */
3556 static int vmx_handle_exit(struct kvm_vcpu *vcpu)
3557 {
3558         struct vcpu_vmx *vmx = to_vmx(vcpu);
3559         u32 exit_reason = vmx->exit_reason;
3560         u32 vectoring_info = vmx->idt_vectoring_info;
3561
3562         trace_kvm_exit(exit_reason, kvm_rip_read(vcpu));
3563
3564         /* If guest state is invalid, start emulating */
3565         if (vmx->emulation_required && emulate_invalid_guest_state)
3566                 return handle_invalid_guest_state(vcpu);
3567
3568         /* Access CR3 don't cause VMExit in paging mode, so we need
3569          * to sync with guest real CR3. */
3570         if (enable_ept && is_paging(vcpu))
3571                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
3572
3573         if (unlikely(vmx->fail)) {
3574                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3575                 vcpu->run->fail_entry.hardware_entry_failure_reason
3576                         = vmcs_read32(VM_INSTRUCTION_ERROR);
3577                 return 0;
3578         }
3579
3580         if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
3581                         (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
3582                         exit_reason != EXIT_REASON_EPT_VIOLATION &&
3583                         exit_reason != EXIT_REASON_TASK_SWITCH))
3584                 printk(KERN_WARNING "%s: unexpected, valid vectoring info "
3585                        "(0x%x) and exit reason is 0x%x\n",
3586                        __func__, vectoring_info, exit_reason);
3587
3588         if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) {
3589                 if (vmx_interrupt_allowed(vcpu)) {
3590                         vmx->soft_vnmi_blocked = 0;
3591                 } else if (vmx->vnmi_blocked_time > 1000000000LL &&
3592                            vcpu->arch.nmi_pending) {
3593                         /*
3594                          * This CPU don't support us in finding the end of an
3595                          * NMI-blocked window if the guest runs with IRQs
3596                          * disabled. So we pull the trigger after 1 s of
3597                          * futile waiting, but inform the user about this.
3598                          */
3599                         printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
3600                                "state on VCPU %d after 1 s timeout\n",
3601                                __func__, vcpu->vcpu_id);
3602                         vmx->soft_vnmi_blocked = 0;
3603                 }
3604         }
3605
3606         if (exit_reason < kvm_vmx_max_exit_handlers
3607             && kvm_vmx_exit_handlers[exit_reason])
3608                 return kvm_vmx_exit_handlers[exit_reason](vcpu);
3609         else {
3610                 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3611                 vcpu->run->hw.hardware_exit_reason = exit_reason;
3612         }
3613         return 0;
3614 }
3615
3616 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3617 {
3618         if (irr == -1 || tpr < irr) {
3619                 vmcs_write32(TPR_THRESHOLD, 0);
3620                 return;
3621         }
3622
3623         vmcs_write32(TPR_THRESHOLD, irr);
3624 }
3625
3626 static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
3627 {
3628         u32 exit_intr_info;
3629         u32 idt_vectoring_info = vmx->idt_vectoring_info;
3630         bool unblock_nmi;
3631         u8 vector;
3632         int type;
3633         bool idtv_info_valid;
3634
3635         exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
3636
3637         vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
3638
3639         /* Handle machine checks before interrupts are enabled */
3640         if ((vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
3641             || (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI
3642                 && is_machine_check(exit_intr_info)))
3643                 kvm_machine_check();
3644
3645         /* We need to handle NMIs before interrupts are enabled */
3646         if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
3647             (exit_intr_info & INTR_INFO_VALID_MASK))
3648                 asm("int $2");
3649
3650         idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3651
3652         if (cpu_has_virtual_nmis()) {
3653                 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
3654                 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
3655                 /*
3656                  * SDM 3: 27.7.1.2 (September 2008)
3657                  * Re-set bit "block by NMI" before VM entry if vmexit caused by
3658                  * a guest IRET fault.
3659                  * SDM 3: 23.2.2 (September 2008)
3660                  * Bit 12 is undefined in any of the following cases:
3661                  *  If the VM exit sets the valid bit in the IDT-vectoring
3662                  *   information field.
3663                  *  If the VM exit is due to a double fault.
3664                  */
3665                 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
3666                     vector != DF_VECTOR && !idtv_info_valid)
3667                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3668                                       GUEST_INTR_STATE_NMI);
3669         } else if (unlikely(vmx->soft_vnmi_blocked))
3670                 vmx->vnmi_blocked_time +=
3671                         ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
3672
3673         vmx->vcpu.arch.nmi_injected = false;
3674         kvm_clear_exception_queue(&vmx->vcpu);
3675         kvm_clear_interrupt_queue(&vmx->vcpu);
3676
3677         if (!idtv_info_valid)
3678                 return;
3679
3680         vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
3681         type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
3682
3683         switch (type) {
3684         case INTR_TYPE_NMI_INTR:
3685                 vmx->vcpu.arch.nmi_injected = true;
3686                 /*
3687                  * SDM 3: 27.7.1.2 (September 2008)
3688                  * Clear bit "block by NMI" before VM entry if a NMI
3689                  * delivery faulted.
3690                  */
3691                 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
3692                                 GUEST_INTR_STATE_NMI);
3693                 break;
3694         case INTR_TYPE_SOFT_EXCEPTION:
3695                 vmx->vcpu.arch.event_exit_inst_len =
3696                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3697                 /* fall through */
3698         case INTR_TYPE_HARD_EXCEPTION:
3699                 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
3700                         u32 err = vmcs_read32(IDT_VECTORING_ERROR_CODE);
3701                         kvm_queue_exception_e(&vmx->vcpu, vector, err);
3702                 } else
3703                         kvm_queue_exception(&vmx->vcpu, vector);
3704                 break;
3705         case INTR_TYPE_SOFT_INTR:
3706                 vmx->vcpu.arch.event_exit_inst_len =
3707                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3708                 /* fall through */
3709         case INTR_TYPE_EXT_INTR:
3710                 kvm_queue_interrupt(&vmx->vcpu, vector,
3711                         type == INTR_TYPE_SOFT_INTR);
3712                 break;
3713         default:
3714                 break;
3715         }
3716 }
3717
3718 /*
3719  * Failure to inject an interrupt should give us the information
3720  * in IDT_VECTORING_INFO_FIELD.  However, if the failure occurs
3721  * when fetching the interrupt redirection bitmap in the real-mode
3722  * tss, this doesn't happen.  So we do it ourselves.
3723  */
3724 static void fixup_rmode_irq(struct vcpu_vmx *vmx)
3725 {
3726         vmx->rmode.irq.pending = 0;
3727         if (kvm_rip_read(&vmx->vcpu) + 1 != vmx->rmode.irq.rip)
3728                 return;
3729         kvm_rip_write(&vmx->vcpu, vmx->rmode.irq.rip);
3730         if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) {
3731                 vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK;
3732                 vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR;
3733                 return;
3734         }
3735         vmx->idt_vectoring_info =
3736                 VECTORING_INFO_VALID_MASK
3737                 | INTR_TYPE_EXT_INTR
3738                 | vmx->rmode.irq.vector;
3739 }
3740
3741 #ifdef CONFIG_X86_64
3742 #define R "r"
3743 #define Q "q"
3744 #else
3745 #define R "e"
3746 #define Q "l"
3747 #endif
3748
3749 static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
3750 {
3751         struct vcpu_vmx *vmx = to_vmx(vcpu);
3752
3753         /* Record the guest's net vcpu time for enforced NMI injections. */
3754         if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
3755                 vmx->entry_time = ktime_get();
3756
3757         /* Don't enter VMX if guest state is invalid, let the exit handler
3758            start emulation until we arrive back to a valid state */
3759         if (vmx->emulation_required && emulate_invalid_guest_state)
3760                 return;
3761
3762         if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
3763                 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
3764         if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
3765                 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
3766
3767         /* When single-stepping over STI and MOV SS, we must clear the
3768          * corresponding interruptibility bits in the guest state. Otherwise
3769          * vmentry fails as it then expects bit 14 (BS) in pending debug
3770          * exceptions being set, but that's not correct for the guest debugging
3771          * case. */
3772         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
3773                 vmx_set_interrupt_shadow(vcpu, 0);
3774
3775         /*
3776          * Loading guest fpu may have cleared host cr0.ts
3777          */
3778         vmcs_writel(HOST_CR0, read_cr0());
3779
3780         if (vcpu->arch.switch_db_regs)
3781                 set_debugreg(vcpu->arch.dr6, 6);
3782
3783         asm(
3784                 /* Store host registers */
3785                 "push %%"R"dx; push %%"R"bp;"
3786                 "push %%"R"cx \n\t"
3787                 "cmp %%"R"sp, %c[host_rsp](%0) \n\t"
3788                 "je 1f \n\t"
3789                 "mov %%"R"sp, %c[host_rsp](%0) \n\t"
3790                 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
3791                 "1: \n\t"
3792                 /* Reload cr2 if changed */
3793                 "mov %c[cr2](%0), %%"R"ax \n\t"
3794                 "mov %%cr2, %%"R"dx \n\t"
3795                 "cmp %%"R"ax, %%"R"dx \n\t"
3796                 "je 2f \n\t"
3797                 "mov %%"R"ax, %%cr2 \n\t"
3798                 "2: \n\t"
3799                 /* Check if vmlaunch of vmresume is needed */
3800                 "cmpl $0, %c[launched](%0) \n\t"
3801                 /* Load guest registers.  Don't clobber flags. */
3802                 "mov %c[rax](%0), %%"R"ax \n\t"
3803                 "mov %c[rbx](%0), %%"R"bx \n\t"
3804                 "mov %c[rdx](%0), %%"R"dx \n\t"
3805                 "mov %c[rsi](%0), %%"R"si \n\t"
3806                 "mov %c[rdi](%0), %%"R"di \n\t"
3807                 "mov %c[rbp](%0), %%"R"bp \n\t"
3808 #ifdef CONFIG_X86_64
3809                 "mov %c[r8](%0),  %%r8  \n\t"
3810                 "mov %c[r9](%0),  %%r9  \n\t"
3811                 "mov %c[r10](%0), %%r10 \n\t"
3812                 "mov %c[r11](%0), %%r11 \n\t"
3813                 "mov %c[r12](%0), %%r12 \n\t"
3814                 "mov %c[r13](%0), %%r13 \n\t"
3815                 "mov %c[r14](%0), %%r14 \n\t"
3816                 "mov %c[r15](%0), %%r15 \n\t"
3817 #endif
3818                 "mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */
3819
3820                 /* Enter guest mode */
3821                 "jne .Llaunched \n\t"
3822                 __ex(ASM_VMX_VMLAUNCH) "\n\t"
3823                 "jmp .Lkvm_vmx_return \n\t"
3824                 ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
3825                 ".Lkvm_vmx_return: "
3826                 /* Save guest registers, load host registers, keep flags */
3827                 "xchg %0,     (%%"R"sp) \n\t"
3828                 "mov %%"R"ax, %c[rax](%0) \n\t"
3829                 "mov %%"R"bx, %c[rbx](%0) \n\t"
3830                 "push"Q" (%%"R"sp); pop"Q" %c[rcx](%0) \n\t"
3831                 "mov %%"R"dx, %c[rdx](%0) \n\t"
3832                 "mov %%"R"si, %c[rsi](%0) \n\t"
3833                 "mov %%"R"di, %c[rdi](%0) \n\t"
3834                 "mov %%"R"bp, %c[rbp](%0) \n\t"
3835 #ifdef CONFIG_X86_64
3836                 "mov %%r8,  %c[r8](%0) \n\t"
3837                 "mov %%r9,  %c[r9](%0) \n\t"
3838                 "mov %%r10, %c[r10](%0) \n\t"
3839                 "mov %%r11, %c[r11](%0) \n\t"
3840                 "mov %%r12, %c[r12](%0) \n\t"
3841                 "mov %%r13, %c[r13](%0) \n\t"
3842                 "mov %%r14, %c[r14](%0) \n\t"
3843                 "mov %%r15, %c[r15](%0) \n\t"
3844 #endif
3845                 "mov %%cr2, %%"R"ax   \n\t"
3846                 "mov %%"R"ax, %c[cr2](%0) \n\t"
3847
3848                 "pop  %%"R"bp; pop  %%"R"bp; pop  %%"R"dx \n\t"
3849                 "setbe %c[fail](%0) \n\t"
3850               : : "c"(vmx), "d"((unsigned long)HOST_RSP),
3851                 [launched]"i"(offsetof(struct vcpu_vmx, launched)),
3852                 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
3853                 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
3854                 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
3855                 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
3856                 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
3857                 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
3858                 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
3859                 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
3860                 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
3861 #ifdef CONFIG_X86_64
3862                 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
3863                 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
3864                 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
3865                 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
3866                 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
3867                 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
3868                 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
3869                 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
3870 #endif
3871                 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2))
3872               : "cc", "memory"
3873                 , R"bx", R"di", R"si"
3874 #ifdef CONFIG_X86_64
3875                 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
3876 #endif
3877               );
3878
3879         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
3880                                   | (1 << VCPU_EXREG_PDPTR));
3881         vcpu->arch.regs_dirty = 0;
3882
3883         if (vcpu->arch.switch_db_regs)
3884                 get_debugreg(vcpu->arch.dr6, 6);
3885
3886         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
3887         if (vmx->rmode.irq.pending)
3888                 fixup_rmode_irq(vmx);
3889
3890         asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
3891         vmx->launched = 1;
3892
3893         vmx_complete_interrupts(vmx);
3894 }
3895
3896 #undef R
3897 #undef Q
3898
3899 static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
3900 {
3901         struct vcpu_vmx *vmx = to_vmx(vcpu);
3902
3903         if (vmx->vmcs) {
3904                 vcpu_clear(vmx);
3905                 free_vmcs(vmx->vmcs);
3906                 vmx->vmcs = NULL;
3907         }
3908 }
3909
3910 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
3911 {
3912         struct vcpu_vmx *vmx = to_vmx(vcpu);
3913
3914         spin_lock(&vmx_vpid_lock);
3915         if (vmx->vpid != 0)
3916                 __clear_bit(vmx->vpid, vmx_vpid_bitmap);
3917         spin_unlock(&vmx_vpid_lock);
3918         vmx_free_vmcs(vcpu);
3919         kfree(vmx->guest_msrs);
3920         kvm_vcpu_uninit(vcpu);
3921         kmem_cache_free(kvm_vcpu_cache, vmx);
3922 }
3923
3924 static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
3925 {
3926         int err;
3927         struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
3928         int cpu;
3929
3930         if (!vmx)
3931                 return ERR_PTR(-ENOMEM);
3932
3933         allocate_vpid(vmx);
3934
3935         err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
3936         if (err)
3937                 goto free_vcpu;
3938
3939         vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
3940         if (!vmx->guest_msrs) {
3941                 err = -ENOMEM;
3942                 goto uninit_vcpu;
3943         }
3944
3945         vmx->vmcs = alloc_vmcs();
3946         if (!vmx->vmcs)
3947                 goto free_msrs;
3948
3949         vmcs_clear(vmx->vmcs);
3950
3951         cpu = get_cpu();
3952         vmx_vcpu_load(&vmx->vcpu, cpu);
3953         err = vmx_vcpu_setup(vmx);
3954         vmx_vcpu_put(&vmx->vcpu);
3955         put_cpu();
3956         if (err)
3957                 goto free_vmcs;
3958         if (vm_need_virtualize_apic_accesses(kvm))
3959                 if (alloc_apic_access_page(kvm) != 0)
3960                         goto free_vmcs;
3961
3962         if (enable_ept) {
3963                 if (!kvm->arch.ept_identity_map_addr)
3964                         kvm->arch.ept_identity_map_addr =
3965                                 VMX_EPT_IDENTITY_PAGETABLE_ADDR;
3966                 if (alloc_identity_pagetable(kvm) != 0)
3967                         goto free_vmcs;
3968         }
3969
3970         return &vmx->vcpu;
3971
3972 free_vmcs:
3973         free_vmcs(vmx->vmcs);
3974 free_msrs:
3975         kfree(vmx->guest_msrs);
3976 uninit_vcpu:
3977         kvm_vcpu_uninit(&vmx->vcpu);
3978 free_vcpu:
3979         kmem_cache_free(kvm_vcpu_cache, vmx);
3980         return ERR_PTR(err);
3981 }
3982
3983 static void __init vmx_check_processor_compat(void *rtn)
3984 {
3985         struct vmcs_config vmcs_conf;
3986
3987         *(int *)rtn = 0;
3988         if (setup_vmcs_config(&vmcs_conf) < 0)
3989                 *(int *)rtn = -EIO;
3990         if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
3991                 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
3992                                 smp_processor_id());
3993                 *(int *)rtn = -EIO;
3994         }
3995 }
3996
3997 static int get_ept_level(void)
3998 {
3999         return VMX_EPT_DEFAULT_GAW + 1;
4000 }
4001
4002 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
4003 {
4004         u64 ret;
4005
4006         /* For VT-d and EPT combination
4007          * 1. MMIO: always map as UC
4008          * 2. EPT with VT-d:
4009          *   a. VT-d without snooping control feature: can't guarantee the
4010          *      result, try to trust guest.
4011          *   b. VT-d with snooping control feature: snooping control feature of
4012          *      VT-d engine can guarantee the cache correctness. Just set it
4013          *      to WB to keep consistent with host. So the same as item 3.
4014          * 3. EPT without VT-d: always map as WB and set IGMT=1 to keep
4015          *    consistent with host MTRR
4016          */
4017         if (is_mmio)
4018                 ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
4019         else if (vcpu->kvm->arch.iommu_domain &&
4020                 !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
4021                 ret = kvm_get_guest_memory_type(vcpu, gfn) <<
4022                       VMX_EPT_MT_EPTE_SHIFT;
4023         else
4024                 ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
4025                         | VMX_EPT_IGMT_BIT;
4026
4027         return ret;
4028 }
4029
4030 #define _ER(x) { EXIT_REASON_##x, #x }
4031
4032 static const struct trace_print_flags vmx_exit_reasons_str[] = {
4033         _ER(EXCEPTION_NMI),
4034         _ER(EXTERNAL_INTERRUPT),
4035         _ER(TRIPLE_FAULT),
4036         _ER(PENDING_INTERRUPT),
4037         _ER(NMI_WINDOW),
4038         _ER(TASK_SWITCH),
4039         _ER(CPUID),
4040         _ER(HLT),
4041         _ER(INVLPG),
4042         _ER(RDPMC),
4043         _ER(RDTSC),
4044         _ER(VMCALL),
4045         _ER(VMCLEAR),
4046         _ER(VMLAUNCH),
4047         _ER(VMPTRLD),
4048         _ER(VMPTRST),
4049         _ER(VMREAD),
4050         _ER(VMRESUME),
4051         _ER(VMWRITE),
4052         _ER(VMOFF),
4053         _ER(VMON),
4054         _ER(CR_ACCESS),
4055         _ER(DR_ACCESS),
4056         _ER(IO_INSTRUCTION),
4057         _ER(MSR_READ),
4058         _ER(MSR_WRITE),
4059         _ER(MWAIT_INSTRUCTION),
4060         _ER(MONITOR_INSTRUCTION),
4061         _ER(PAUSE_INSTRUCTION),
4062         _ER(MCE_DURING_VMENTRY),
4063         _ER(TPR_BELOW_THRESHOLD),
4064         _ER(APIC_ACCESS),
4065         _ER(EPT_VIOLATION),
4066         _ER(EPT_MISCONFIG),
4067         _ER(WBINVD),
4068         { -1, NULL }
4069 };
4070
4071 #undef _ER
4072
4073 static int vmx_get_lpage_level(void)
4074 {
4075         if (enable_ept && !cpu_has_vmx_ept_1g_page())
4076                 return PT_DIRECTORY_LEVEL;
4077         else
4078                 /* For shadow and EPT supported 1GB page */
4079                 return PT_PDPE_LEVEL;
4080 }
4081
4082 static inline u32 bit(int bitno)
4083 {
4084         return 1 << (bitno & 31);
4085 }
4086
4087 static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
4088 {
4089         struct kvm_cpuid_entry2 *best;
4090         struct vcpu_vmx *vmx = to_vmx(vcpu);
4091         u32 exec_control;
4092
4093         vmx->rdtscp_enabled = false;
4094         if (vmx_rdtscp_supported()) {
4095                 exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
4096                 if (exec_control & SECONDARY_EXEC_RDTSCP) {
4097                         best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
4098                         if (best && (best->edx & bit(X86_FEATURE_RDTSCP)))
4099                                 vmx->rdtscp_enabled = true;
4100                         else {
4101                                 exec_control &= ~SECONDARY_EXEC_RDTSCP;
4102                                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
4103                                                 exec_control);
4104                         }
4105                 }
4106         }
4107 }
4108
4109 static struct kvm_x86_ops vmx_x86_ops = {
4110         .cpu_has_kvm_support = cpu_has_kvm_support,
4111         .disabled_by_bios = vmx_disabled_by_bios,
4112         .hardware_setup = hardware_setup,
4113         .hardware_unsetup = hardware_unsetup,
4114         .check_processor_compatibility = vmx_check_processor_compat,
4115         .hardware_enable = hardware_enable,
4116         .hardware_disable = hardware_disable,
4117         .cpu_has_accelerated_tpr = report_flexpriority,
4118
4119         .vcpu_create = vmx_create_vcpu,
4120         .vcpu_free = vmx_free_vcpu,
4121         .vcpu_reset = vmx_vcpu_reset,
4122
4123         .prepare_guest_switch = vmx_save_host_state,
4124         .vcpu_load = vmx_vcpu_load,
4125         .vcpu_put = vmx_vcpu_put,
4126
4127         .set_guest_debug = set_guest_debug,
4128         .get_msr = vmx_get_msr,
4129         .set_msr = vmx_set_msr,
4130         .get_segment_base = vmx_get_segment_base,
4131         .get_segment = vmx_get_segment,
4132         .set_segment = vmx_set_segment,
4133         .get_cpl = vmx_get_cpl,
4134         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
4135         .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
4136         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
4137         .set_cr0 = vmx_set_cr0,
4138         .set_cr3 = vmx_set_cr3,
4139         .set_cr4 = vmx_set_cr4,
4140         .set_efer = vmx_set_efer,
4141         .get_idt = vmx_get_idt,
4142         .set_idt = vmx_set_idt,
4143         .get_gdt = vmx_get_gdt,
4144         .set_gdt = vmx_set_gdt,
4145         .cache_reg = vmx_cache_reg,
4146         .get_rflags = vmx_get_rflags,
4147         .set_rflags = vmx_set_rflags,
4148         .fpu_deactivate = vmx_fpu_deactivate,
4149
4150         .tlb_flush = vmx_flush_tlb,
4151
4152         .run = vmx_vcpu_run,
4153         .handle_exit = vmx_handle_exit,
4154         .skip_emulated_instruction = skip_emulated_instruction,
4155         .set_interrupt_shadow = vmx_set_interrupt_shadow,
4156         .get_interrupt_shadow = vmx_get_interrupt_shadow,
4157         .patch_hypercall = vmx_patch_hypercall,
4158         .set_irq = vmx_inject_irq,
4159         .set_nmi = vmx_inject_nmi,
4160         .queue_exception = vmx_queue_exception,
4161         .interrupt_allowed = vmx_interrupt_allowed,
4162         .nmi_allowed = vmx_nmi_allowed,
4163         .get_nmi_mask = vmx_get_nmi_mask,
4164         .set_nmi_mask = vmx_set_nmi_mask,
4165         .enable_nmi_window = enable_nmi_window,
4166         .enable_irq_window = enable_irq_window,
4167         .update_cr8_intercept = update_cr8_intercept,
4168
4169         .set_tss_addr = vmx_set_tss_addr,
4170         .get_tdp_level = get_ept_level,
4171         .get_mt_mask = vmx_get_mt_mask,
4172
4173         .exit_reasons_str = vmx_exit_reasons_str,
4174         .get_lpage_level = vmx_get_lpage_level,
4175
4176         .cpuid_update = vmx_cpuid_update,
4177
4178         .rdtscp_supported = vmx_rdtscp_supported,
4179 };
4180
4181 static int __init vmx_init(void)
4182 {
4183         int r, i;
4184
4185         rdmsrl_safe(MSR_EFER, &host_efer);
4186
4187         for (i = 0; i < NR_VMX_MSR; ++i)
4188                 kvm_define_shared_msr(i, vmx_msr_index[i]);
4189
4190         vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
4191         if (!vmx_io_bitmap_a)
4192                 return -ENOMEM;
4193
4194         vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
4195         if (!vmx_io_bitmap_b) {
4196                 r = -ENOMEM;
4197                 goto out;
4198         }
4199
4200         vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
4201         if (!vmx_msr_bitmap_legacy) {
4202                 r = -ENOMEM;
4203                 goto out1;
4204         }
4205
4206         vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
4207         if (!vmx_msr_bitmap_longmode) {
4208                 r = -ENOMEM;
4209                 goto out2;
4210         }
4211
4212         /*
4213          * Allow direct access to the PC debug port (it is often used for I/O
4214          * delays, but the vmexits simply slow things down).
4215          */
4216         memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
4217         clear_bit(0x80, vmx_io_bitmap_a);
4218
4219         memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
4220
4221         memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
4222         memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
4223
4224         set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
4225
4226         r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE);
4227         if (r)
4228                 goto out3;
4229
4230         vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
4231         vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
4232         vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
4233         vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
4234         vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
4235         vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
4236
4237         if (enable_ept) {
4238                 bypass_guest_pf = 0;
4239                 kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
4240                         VMX_EPT_WRITABLE_MASK);
4241                 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
4242                                 VMX_EPT_EXECUTABLE_MASK);
4243                 kvm_enable_tdp();
4244         } else
4245                 kvm_disable_tdp();
4246
4247         if (bypass_guest_pf)
4248                 kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull);
4249
4250         return 0;
4251
4252 out3:
4253         free_page((unsigned long)vmx_msr_bitmap_longmode);
4254 out2:
4255         free_page((unsigned long)vmx_msr_bitmap_legacy);
4256 out1:
4257         free_page((unsigned long)vmx_io_bitmap_b);
4258 out:
4259         free_page((unsigned long)vmx_io_bitmap_a);
4260         return r;
4261 }
4262
4263 static void __exit vmx_exit(void)
4264 {
4265         free_page((unsigned long)vmx_msr_bitmap_legacy);
4266         free_page((unsigned long)vmx_msr_bitmap_longmode);
4267         free_page((unsigned long)vmx_io_bitmap_b);
4268         free_page((unsigned long)vmx_io_bitmap_a);
4269
4270         kvm_exit();
4271 }
4272
4273 module_init(vmx_init)
4274 module_exit(vmx_exit)