SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19 #include "x86.h"
  20 #include "x86_emulate.h"
  21 #include "irq.h"
  22
  23 #include <linux/kvm.h>
  24 #include <linux/module.h>
  25 #include <linux/errno.h>
  26 #include <linux/percpu.h>
  27 #include <linux/gfp.h>
  28 #include <linux/mm.h>
  29 #include <linux/miscdevice.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/reboot.h>
  32 #include <linux/debugfs.h>
  33 #include <linux/highmem.h>
  34 #include <linux/file.h>
  35 #include <linux/sysdev.h>
  36 #include <linux/cpu.h>
  37 #include <linux/sched.h>
  38 #include <linux/cpumask.h>
  39 #include <linux/smp.h>
  40 #include <linux/anon_inodes.h>
  41 #include <linux/profile.h>
  42 #include <linux/kvm_para.h>
  43 #include <linux/pagemap.h>
  44 #include <linux/mman.h>
  45
  46 #include <asm/processor.h>
  47 #include <asm/msr.h>
  48 #include <asm/io.h>
  49 #include <asm/uaccess.h>
  50 #include <asm/desc.h>
  51
  52 MODULE_AUTHOR("Qumranet");
  53 MODULE_LICENSE("GPL");
  54
  55 static DEFINE_SPINLOCK(kvm_lock);
  56 static LIST_HEAD(vm_list);
  57
  58 static cpumask_t cpus_hardware_enabled;
  59
  60 struct kvm_x86_ops *kvm_x86_ops;
  61 struct kmem_cache *kvm_vcpu_cache;
  62 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  63
  64 static __read_mostly struct preempt_ops kvm_preempt_ops;
  65
  66 static struct dentry *debugfs_dir;
  67
  68 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  69                            unsigned long arg);
  70
  71 static inline int valid_vcpu(int n)
  72 {
  73         return likely(n >= 0 && n < KVM_MAX_VCPUS);
  74 }
  75
  76 /*
  77  * Switches to specified vcpu, until a matching vcpu_put()
  78  */
  79 void vcpu_load(struct kvm_vcpu *vcpu)
  80 {
  81         int cpu;
  82
  83         mutex_lock(&vcpu->mutex);
  84         cpu = get_cpu();
  85         preempt_notifier_register(&vcpu->preempt_notifier);
  86         kvm_arch_vcpu_load(vcpu, cpu);
  87         put_cpu();
  88 }
  89
  90 void vcpu_put(struct kvm_vcpu *vcpu)
  91 {
  92         preempt_disable();
  93         kvm_arch_vcpu_put(vcpu);
  94         preempt_notifier_unregister(&vcpu->preempt_notifier);
  95         preempt_enable();
  96         mutex_unlock(&vcpu->mutex);
  97 }
  98
  99 static void ack_flush(void *_completed)
 100 {
 101 }
 102
 103 void kvm_flush_remote_tlbs(struct kvm *kvm)
 104 {
 105         int i, cpu;
 106         cpumask_t cpus;
 107         struct kvm_vcpu *vcpu;
 108
 109         cpus_clear(cpus);
 110         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 111                 vcpu = kvm->vcpus[i];
 112                 if (!vcpu)
 113                         continue;
 114                 if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
 115                         continue;
 116                 cpu = vcpu->cpu;
 117                 if (cpu != -1 && cpu != raw_smp_processor_id())
 118                         cpu_set(cpu, cpus);
 119         }
 120         smp_call_function_mask(cpus, ack_flush, NULL, 1);
 121 }
 122
 123 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 124 {
 125         struct page *page;
 126         int r;
 127
 128         mutex_init(&vcpu->mutex);
 129         vcpu->cpu = -1;
 130         vcpu->mmu.root_hpa = INVALID_PAGE;
 131         vcpu->kvm = kvm;
 132         vcpu->vcpu_id = id;
 133         if (!irqchip_in_kernel(kvm) || id == 0)
 134                 vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
 135         else
 136                 vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
 137         init_waitqueue_head(&vcpu->wq);
 138
 139         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 140         if (!page) {
 141                 r = -ENOMEM;
 142                 goto fail;
 143         }
 144         vcpu->run = page_address(page);
 145
 146         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 147         if (!page) {
 148                 r = -ENOMEM;
 149                 goto fail_free_run;
 150         }
 151         vcpu->pio_data = page_address(page);
 152
 153         r = kvm_mmu_create(vcpu);
 154         if (r < 0)
 155                 goto fail_free_pio_data;
 156
 157         if (irqchip_in_kernel(kvm)) {
 158                 r = kvm_create_lapic(vcpu);
 159                 if (r < 0)
 160                         goto fail_mmu_destroy;
 161         }
 162
 163         return 0;
 164
 165 fail_mmu_destroy:
 166         kvm_mmu_destroy(vcpu);
 167 fail_free_pio_data:
 168         free_page((unsigned long)vcpu->pio_data);
 169 fail_free_run:
 170         free_page((unsigned long)vcpu->run);
 171 fail:
 172         return r;
 173 }
 174 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 175
 176 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 177 {
 178         kvm_free_lapic(vcpu);
 179         kvm_mmu_destroy(vcpu);
 180         free_page((unsigned long)vcpu->pio_data);
 181         free_page((unsigned long)vcpu->run);
 182 }
 183 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 184
 185 static struct kvm *kvm_create_vm(void)
 186 {
 187         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 188
 189         if (!kvm)
 190                 return ERR_PTR(-ENOMEM);
 191
 192         kvm_io_bus_init(&kvm->pio_bus);
 193         mutex_init(&kvm->lock);
 194         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 195         kvm_io_bus_init(&kvm->mmio_bus);
 196         spin_lock(&kvm_lock);
 197         list_add(&kvm->vm_list, &vm_list);
 198         spin_unlock(&kvm_lock);
 199         return kvm;
 200 }
 201
 202 /*
 203  * Free any memory in @free but not in @dont.
 204  */
 205 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 206                                   struct kvm_memory_slot *dont)
 207 {
 208         if (!dont || free->rmap != dont->rmap)
 209                 vfree(free->rmap);
 210
 211         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 212                 vfree(free->dirty_bitmap);
 213
 214         free->npages = 0;
 215         free->dirty_bitmap = NULL;
 216         free->rmap = NULL;
 217 }
 218
 219 static void kvm_free_physmem(struct kvm *kvm)
 220 {
 221         int i;
 222
 223         for (i = 0; i < kvm->nmemslots; ++i)
 224                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 225 }
 226
 227 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
 228 {
 229         vcpu_load(vcpu);
 230         kvm_mmu_unload(vcpu);
 231         vcpu_put(vcpu);
 232 }
 233
 234 static void kvm_free_vcpus(struct kvm *kvm)
 235 {
 236         unsigned int i;
 237
 238         /*
 239          * Unpin any mmu pages first.
 240          */
 241         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 242                 if (kvm->vcpus[i])
 243                         kvm_unload_vcpu_mmu(kvm->vcpus[i]);
 244         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 245                 if (kvm->vcpus[i]) {
 246                         kvm_x86_ops->vcpu_free(kvm->vcpus[i]);
 247                         kvm->vcpus[i] = NULL;
 248                 }
 249         }
 250
 251 }
 252
 253 static void kvm_destroy_vm(struct kvm *kvm)
 254 {
 255         spin_lock(&kvm_lock);
 256         list_del(&kvm->vm_list);
 257         spin_unlock(&kvm_lock);
 258         kvm_io_bus_destroy(&kvm->pio_bus);
 259         kvm_io_bus_destroy(&kvm->mmio_bus);
 260         kfree(kvm->vpic);
 261         kfree(kvm->vioapic);
 262         kvm_free_vcpus(kvm);
 263         kvm_free_physmem(kvm);
 264         kfree(kvm);
 265 }
 266
 267 static int kvm_vm_release(struct inode *inode, struct file *filp)
 268 {
 269         struct kvm *kvm = filp->private_data;
 270
 271         kvm_destroy_vm(kvm);
 272         return 0;
 273 }
 274
 275 /*
 276  * Allocate some memory and give it an address in the guest physical address
 277  * space.
 278  *
 279  * Discontiguous memory is allowed, mostly for framebuffers.
 280  *
 281  * Must be called holding kvm->lock.
 282  */
 283 int __kvm_set_memory_region(struct kvm *kvm,
 284                             struct kvm_userspace_memory_region *mem,
 285                             int user_alloc)
 286 {
 287         int r;
 288         gfn_t base_gfn;
 289         unsigned long npages;
 290         unsigned long i;
 291         struct kvm_memory_slot *memslot;
 292         struct kvm_memory_slot old, new;
 293
 294         r = -EINVAL;
 295         /* General sanity checks */
 296         if (mem->memory_size & (PAGE_SIZE - 1))
 297                 goto out;
 298         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 299                 goto out;
 300         if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 301                 goto out;
 302         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 303                 goto out;
 304
 305         memslot = &kvm->memslots[mem->slot];
 306         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 307         npages = mem->memory_size >> PAGE_SHIFT;
 308
 309         if (!npages)
 310                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 311
 312         new = old = *memslot;
 313
 314         new.base_gfn = base_gfn;
 315         new.npages = npages;
 316         new.flags = mem->flags;
 317
 318         /* Disallow changing a memory slot's size. */
 319         r = -EINVAL;
 320         if (npages && old.npages && npages != old.npages)
 321                 goto out_free;
 322
 323         /* Check for overlaps */
 324         r = -EEXIST;
 325         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 326                 struct kvm_memory_slot *s = &kvm->memslots[i];
 327
 328                 if (s == memslot)
 329                         continue;
 330                 if (!((base_gfn + npages <= s->base_gfn) ||
 331                       (base_gfn >= s->base_gfn + s->npages)))
 332                         goto out_free;
 333         }
 334
 335         /* Free page dirty bitmap if unneeded */
 336         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 337                 new.dirty_bitmap = NULL;
 338
 339         r = -ENOMEM;
 340
 341         /* Allocate if a slot is being created */
 342         if (npages && !new.rmap) {
 343                 new.rmap = vmalloc(npages * sizeof(struct page *));
 344
 345                 if (!new.rmap)
 346                         goto out_free;
 347
 348                 memset(new.rmap, 0, npages * sizeof(*new.rmap));
 349
 350                 new.user_alloc = user_alloc;
 351                 if (user_alloc)
 352                         new.userspace_addr = mem->userspace_addr;
 353                 else {
 354                         down_write(&current->mm->mmap_sem);
 355                         new.userspace_addr = do_mmap(NULL, 0,
 356                                                      npages * PAGE_SIZE,
 357                                                      PROT_READ | PROT_WRITE,
 358                                                      MAP_SHARED | MAP_ANONYMOUS,
 359                                                      0);
 360                         up_write(&current->mm->mmap_sem);
 361
 362                         if (IS_ERR((void *)new.userspace_addr))
 363                                 goto out_free;
 364                 }
 365         } else {
 366                 if (!old.user_alloc && old.rmap) {
 367                         int ret;
 368
 369                         down_write(&current->mm->mmap_sem);
 370                         ret = do_munmap(current->mm, old.userspace_addr,
 371                                         old.npages * PAGE_SIZE);
 372                         up_write(&current->mm->mmap_sem);
 373                         if (ret < 0)
 374                                 printk(KERN_WARNING
 375                                        "kvm_vm_ioctl_set_memory_region: "
 376                                        "failed to munmap memory\n");
 377                 }
 378         }
 379
 380         /* Allocate page dirty bitmap if needed */
 381         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 382                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 383
 384                 new.dirty_bitmap = vmalloc(dirty_bytes);
 385                 if (!new.dirty_bitmap)
 386                         goto out_free;
 387                 memset(new.dirty_bitmap, 0, dirty_bytes);
 388         }
 389
 390         if (mem->slot >= kvm->nmemslots)
 391                 kvm->nmemslots = mem->slot + 1;
 392
 393         if (!kvm->n_requested_mmu_pages) {
 394                 unsigned int n_pages;
 395
 396                 if (npages) {
 397                         n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000;
 398                         kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages +
 399                                                  n_pages);
 400                 } else {
 401                         unsigned int nr_mmu_pages;
 402
 403                         n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000;
 404                         nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages;
 405                         nr_mmu_pages = max(nr_mmu_pages,
 406                                         (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
 407                         kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
 408                 }
 409         }
 410
 411         *memslot = new;
 412
 413         kvm_mmu_slot_remove_write_access(kvm, mem->slot);
 414         kvm_flush_remote_tlbs(kvm);
 415
 416         kvm_free_physmem_slot(&old, &new);
 417         return 0;
 418
 419 out_free:
 420         kvm_free_physmem_slot(&new, &old);
 421 out:
 422         return r;
 423
 424 }
 425 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 426
 427 int kvm_set_memory_region(struct kvm *kvm,
 428                           struct kvm_userspace_memory_region *mem,
 429                           int user_alloc)
 430 {
 431         int r;
 432
 433         mutex_lock(&kvm->lock);
 434         r = __kvm_set_memory_region(kvm, mem, user_alloc);
 435         mutex_unlock(&kvm->lock);
 436         return r;
 437 }
 438 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
 439
 440 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 441                                    struct
 442                                    kvm_userspace_memory_region *mem,
 443                                    int user_alloc)
 444 {
 445         if (mem->slot >= KVM_MEMORY_SLOTS)
 446                 return -EINVAL;
 447         return kvm_set_memory_region(kvm, mem, user_alloc);
 448 }
 449
 450 /*
 451  * Get (and clear) the dirty memory log for a memory slot.
 452  */
 453 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 454                                       struct kvm_dirty_log *log)
 455 {
 456         struct kvm_memory_slot *memslot;
 457         int r, i;
 458         int n;
 459         unsigned long any = 0;
 460
 461         mutex_lock(&kvm->lock);
 462
 463         r = -EINVAL;
 464         if (log->slot >= KVM_MEMORY_SLOTS)
 465                 goto out;
 466
 467         memslot = &kvm->memslots[log->slot];
 468         r = -ENOENT;
 469         if (!memslot->dirty_bitmap)
 470                 goto out;
 471
 472         n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 473
 474         for (i = 0; !any && i < n/sizeof(long); ++i)
 475                 any = memslot->dirty_bitmap[i];
 476
 477         r = -EFAULT;
 478         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 479                 goto out;
 480
 481         /* If nothing is dirty, don't bother messing with page tables. */
 482         if (any) {
 483                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
 484                 kvm_flush_remote_tlbs(kvm);
 485                 memset(memslot->dirty_bitmap, 0, n);
 486         }
 487
 488         r = 0;
 489
 490 out:
 491         mutex_unlock(&kvm->lock);
 492         return r;
 493 }
 494
 495 int is_error_page(struct page *page)
 496 {
 497         return page == bad_page;
 498 }
 499 EXPORT_SYMBOL_GPL(is_error_page);
 500
 501 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 502 {
 503         int i;
 504         struct kvm_mem_alias *alias;
 505
 506         for (i = 0; i < kvm->naliases; ++i) {
 507                 alias = &kvm->aliases[i];
 508                 if (gfn >= alias->base_gfn
 509                     && gfn < alias->base_gfn + alias->npages)
 510                         return alias->target_gfn + gfn - alias->base_gfn;
 511         }
 512         return gfn;
 513 }
 514
 515 static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 516 {
 517         int i;
 518
 519         for (i = 0; i < kvm->nmemslots; ++i) {
 520                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 521
 522                 if (gfn >= memslot->base_gfn
 523                     && gfn < memslot->base_gfn + memslot->npages)
 524                         return memslot;
 525         }
 526         return NULL;
 527 }
 528
 529 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 530 {
 531         gfn = unalias_gfn(kvm, gfn);
 532         return __gfn_to_memslot(kvm, gfn);
 533 }
 534
 535 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
 536 {
 537         int i;
 538
 539         gfn = unalias_gfn(kvm, gfn);
 540         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 541                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 542
 543                 if (gfn >= memslot->base_gfn
 544                     && gfn < memslot->base_gfn + memslot->npages)
 545                         return 1;
 546         }
 547         return 0;
 548 }
 549 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
 550
 551 /*
 552  * Requires current->mm->mmap_sem to be held
 553  */
 554 static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn)
 555 {
 556         struct kvm_memory_slot *slot;
 557         struct page *page[1];
 558         int npages;
 559
 560         might_sleep();
 561
 562         gfn = unalias_gfn(kvm, gfn);
 563         slot = __gfn_to_memslot(kvm, gfn);
 564         if (!slot) {
 565                 get_page(bad_page);
 566                 return bad_page;
 567         }
 568
 569         npages = get_user_pages(current, current->mm,
 570                                 slot->userspace_addr
 571                                 + (gfn - slot->base_gfn) * PAGE_SIZE, 1,
 572                                 1, 1, page, NULL);
 573         if (npages != 1) {
 574                 get_page(bad_page);
 575                 return bad_page;
 576         }
 577
 578         return page[0];
 579 }
 580
 581 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
 582 {
 583         struct page *page;
 584
 585         down_read(&current->mm->mmap_sem);
 586         page = __gfn_to_page(kvm, gfn);
 587         up_read(&current->mm->mmap_sem);
 588
 589         return page;
 590 }
 591
 592 EXPORT_SYMBOL_GPL(gfn_to_page);
 593
 594 void kvm_release_page(struct page *page)
 595 {
 596         if (!PageReserved(page))
 597                 SetPageDirty(page);
 598         put_page(page);
 599 }
 600 EXPORT_SYMBOL_GPL(kvm_release_page);
 601
 602 static int next_segment(unsigned long len, int offset)
 603 {
 604         if (len > PAGE_SIZE - offset)
 605                 return PAGE_SIZE - offset;
 606         else
 607                 return len;
 608 }
 609
 610 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
 611                         int len)
 612 {
 613         void *page_virt;
 614         struct page *page;
 615
 616         page = gfn_to_page(kvm, gfn);
 617         if (is_error_page(page)) {
 618                 kvm_release_page(page);
 619                 return -EFAULT;
 620         }
 621         page_virt = kmap_atomic(page, KM_USER0);
 622
 623         memcpy(data, page_virt + offset, len);
 624
 625         kunmap_atomic(page_virt, KM_USER0);
 626         kvm_release_page(page);
 627         return 0;
 628 }
 629 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
 630
 631 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
 632 {
 633         gfn_t gfn = gpa >> PAGE_SHIFT;
 634         int seg;
 635         int offset = offset_in_page(gpa);
 636         int ret;
 637
 638         while ((seg = next_segment(len, offset)) != 0) {
 639                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
 640                 if (ret < 0)
 641                         return ret;
 642                 offset = 0;
 643                 len -= seg;
 644                 data += seg;
 645                 ++gfn;
 646         }
 647         return 0;
 648 }
 649 EXPORT_SYMBOL_GPL(kvm_read_guest);
 650
 651 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
 652                          int offset, int len)
 653 {
 654         void *page_virt;
 655         struct page *page;
 656
 657         page = gfn_to_page(kvm, gfn);
 658         if (is_error_page(page)) {
 659                 kvm_release_page(page);
 660                 return -EFAULT;
 661         }
 662         page_virt = kmap_atomic(page, KM_USER0);
 663
 664         memcpy(page_virt + offset, data, len);
 665
 666         kunmap_atomic(page_virt, KM_USER0);
 667         mark_page_dirty(kvm, gfn);
 668         kvm_release_page(page);
 669         return 0;
 670 }
 671 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
 672
 673 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
 674                     unsigned long len)
 675 {
 676         gfn_t gfn = gpa >> PAGE_SHIFT;
 677         int seg;
 678         int offset = offset_in_page(gpa);
 679         int ret;
 680
 681         while ((seg = next_segment(len, offset)) != 0) {
 682                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
 683                 if (ret < 0)
 684                         return ret;
 685                 offset = 0;
 686                 len -= seg;
 687                 data += seg;
 688                 ++gfn;
 689         }
 690         return 0;
 691 }
 692
 693 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
 694 {
 695         void *page_virt;
 696         struct page *page;
 697
 698         page = gfn_to_page(kvm, gfn);
 699         if (is_error_page(page)) {
 700                 kvm_release_page(page);
 701                 return -EFAULT;
 702         }
 703         page_virt = kmap_atomic(page, KM_USER0);
 704
 705         memset(page_virt + offset, 0, len);
 706
 707         kunmap_atomic(page_virt, KM_USER0);
 708         kvm_release_page(page);
 709         mark_page_dirty(kvm, gfn);
 710         return 0;
 711 }
 712 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
 713
 714 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
 715 {
 716         gfn_t gfn = gpa >> PAGE_SHIFT;
 717         int seg;
 718         int offset = offset_in_page(gpa);
 719         int ret;
 720
 721         while ((seg = next_segment(len, offset)) != 0) {
 722                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
 723                 if (ret < 0)
 724                         return ret;
 725                 offset = 0;
 726                 len -= seg;
 727                 ++gfn;
 728         }
 729         return 0;
 730 }
 731 EXPORT_SYMBOL_GPL(kvm_clear_guest);
 732
 733 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 734 {
 735         struct kvm_memory_slot *memslot;
 736
 737         gfn = unalias_gfn(kvm, gfn);
 738         memslot = __gfn_to_memslot(kvm, gfn);
 739         if (memslot && memslot->dirty_bitmap) {
 740                 unsigned long rel_gfn = gfn - memslot->base_gfn;
 741
 742                 /* avoid RMW */
 743                 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 744                         set_bit(rel_gfn, memslot->dirty_bitmap);
 745         }
 746 }
 747
 748 /*
 749  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 750  */
 751 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 752 {
 753         DECLARE_WAITQUEUE(wait, current);
 754
 755         add_wait_queue(&vcpu->wq, &wait);
 756
 757         /*
 758          * We will block until either an interrupt or a signal wakes us up
 759          */
 760         while (!kvm_cpu_has_interrupt(vcpu)
 761                && !signal_pending(current)
 762                && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE
 763                && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) {
 764                 set_current_state(TASK_INTERRUPTIBLE);
 765                 vcpu_put(vcpu);
 766                 schedule();
 767                 vcpu_load(vcpu);
 768         }
 769
 770         __set_current_state(TASK_RUNNING);
 771         remove_wait_queue(&vcpu->wq, &wait);
 772 }
 773
 774 void kvm_resched(struct kvm_vcpu *vcpu)
 775 {
 776         if (!need_resched())
 777                 return;
 778         cond_resched();
 779 }
 780 EXPORT_SYMBOL_GPL(kvm_resched);
 781
 782 /*
 783  * Translate a guest virtual address to a guest physical address.
 784  */
 785 static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 786                                     struct kvm_translation *tr)
 787 {
 788         unsigned long vaddr = tr->linear_address;
 789         gpa_t gpa;
 790
 791         vcpu_load(vcpu);
 792         mutex_lock(&vcpu->kvm->lock);
 793         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
 794         tr->physical_address = gpa;
 795         tr->valid = gpa != UNMAPPED_GVA;
 796         tr->writeable = 1;
 797         tr->usermode = 0;
 798         mutex_unlock(&vcpu->kvm->lock);
 799         vcpu_put(vcpu);
 800
 801         return 0;
 802 }
 803
 804 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
 805                                     struct kvm_interrupt *irq)
 806 {
 807         if (irq->irq < 0 || irq->irq >= 256)
 808                 return -EINVAL;
 809         if (irqchip_in_kernel(vcpu->kvm))
 810                 return -ENXIO;
 811         vcpu_load(vcpu);
 812
 813         set_bit(irq->irq, vcpu->irq_pending);
 814         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
 815
 816         vcpu_put(vcpu);
 817
 818         return 0;
 819 }
 820
 821 static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
 822                                     unsigned long address,
 823                                     int *type)
 824 {
 825         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
 826         unsigned long pgoff;
 827         struct page *page;
 828
 829         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
 830         if (pgoff == 0)
 831                 page = virt_to_page(vcpu->run);
 832         else if (pgoff == KVM_PIO_PAGE_OFFSET)
 833                 page = virt_to_page(vcpu->pio_data);
 834         else
 835                 return NOPAGE_SIGBUS;
 836         get_page(page);
 837         if (type != NULL)
 838                 *type = VM_FAULT_MINOR;
 839
 840         return page;
 841 }
 842
 843 static struct vm_operations_struct kvm_vcpu_vm_ops = {
 844         .nopage = kvm_vcpu_nopage,
 845 };
 846
 847 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
 848 {
 849         vma->vm_ops = &kvm_vcpu_vm_ops;
 850         return 0;
 851 }
 852
 853 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
 854 {
 855         struct kvm_vcpu *vcpu = filp->private_data;
 856
 857         fput(vcpu->kvm->filp);
 858         return 0;
 859 }
 860
 861 static struct file_operations kvm_vcpu_fops = {
 862         .release        = kvm_vcpu_release,
 863         .unlocked_ioctl = kvm_vcpu_ioctl,
 864         .compat_ioctl   = kvm_vcpu_ioctl,
 865         .mmap           = kvm_vcpu_mmap,
 866 };
 867
 868 /*
 869  * Allocates an inode for the vcpu.
 870  */
 871 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
 872 {
 873         int fd, r;
 874         struct inode *inode;
 875         struct file *file;
 876
 877         r = anon_inode_getfd(&fd, &inode, &file,
 878                              "kvm-vcpu", &kvm_vcpu_fops, vcpu);
 879         if (r)
 880                 return r;
 881         atomic_inc(&vcpu->kvm->filp->f_count);
 882         return fd;
 883 }
 884
 885 /*
 886  * Creates some virtual cpus.  Good luck creating more than one.
 887  */
 888 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
 889 {
 890         int r;
 891         struct kvm_vcpu *vcpu;
 892
 893         if (!valid_vcpu(n))
 894                 return -EINVAL;
 895
 896         vcpu = kvm_x86_ops->vcpu_create(kvm, n);
 897         if (IS_ERR(vcpu))
 898                 return PTR_ERR(vcpu);
 899
 900         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
 901
 902         /* We do fxsave: this must be aligned. */
 903         BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);
 904
 905         vcpu_load(vcpu);
 906         r = kvm_x86_ops->vcpu_reset(vcpu);
 907         if (r == 0)
 908                 r = kvm_mmu_setup(vcpu);
 909         vcpu_put(vcpu);
 910         if (r < 0)
 911                 goto free_vcpu;
 912
 913         mutex_lock(&kvm->lock);
 914         if (kvm->vcpus[n]) {
 915                 r = -EEXIST;
 916                 mutex_unlock(&kvm->lock);
 917                 goto mmu_unload;
 918         }
 919         kvm->vcpus[n] = vcpu;
 920         mutex_unlock(&kvm->lock);
 921
 922         /* Now it's all set up, let userspace reach it */
 923         r = create_vcpu_fd(vcpu);
 924         if (r < 0)
 925                 goto unlink;
 926         return r;
 927
 928 unlink:
 929         mutex_lock(&kvm->lock);
 930         kvm->vcpus[n] = NULL;
 931         mutex_unlock(&kvm->lock);
 932
 933 mmu_unload:
 934         vcpu_load(vcpu);
 935         kvm_mmu_unload(vcpu);
 936         vcpu_put(vcpu);
 937
 938 free_vcpu:
 939         kvm_x86_ops->vcpu_free(vcpu);
 940         return r;
 941 }
 942
 943 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
 944 {
 945         if (sigset) {
 946                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
 947                 vcpu->sigset_active = 1;
 948                 vcpu->sigset = *sigset;
 949         } else
 950                 vcpu->sigset_active = 0;
 951         return 0;
 952 }
 953
 954 static long kvm_vcpu_ioctl(struct file *filp,
 955                            unsigned int ioctl, unsigned long arg)
 956 {
 957         struct kvm_vcpu *vcpu = filp->private_data;
 958         void __user *argp = (void __user *)arg;
 959         int r;
 960
 961         switch (ioctl) {
 962         case KVM_RUN:
 963                 r = -EINVAL;
 964                 if (arg)
 965                         goto out;
 966                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
 967                 break;
 968         case KVM_GET_REGS: {
 969                 struct kvm_regs kvm_regs;
 970
 971                 memset(&kvm_regs, 0, sizeof kvm_regs);
 972                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
 973                 if (r)
 974                         goto out;
 975                 r = -EFAULT;
 976                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
 977                         goto out;
 978                 r = 0;
 979                 break;
 980         }
 981         case KVM_SET_REGS: {
 982                 struct kvm_regs kvm_regs;
 983
 984                 r = -EFAULT;
 985                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
 986                         goto out;
 987                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
 988                 if (r)
 989                         goto out;
 990                 r = 0;
 991                 break;
 992         }
 993         case KVM_GET_SREGS: {
 994                 struct kvm_sregs kvm_sregs;
 995
 996                 memset(&kvm_sregs, 0, sizeof kvm_sregs);
 997                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
 998                 if (r)
 999                         goto out;
1000                 r = -EFAULT;
1001                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
1002                         goto out;
1003                 r = 0;
1004                 break;
1005         }
1006         case KVM_SET_SREGS: {
1007                 struct kvm_sregs kvm_sregs;
1008
1009                 r = -EFAULT;
1010                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
1011                         goto out;
1012                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
1013                 if (r)
1014                         goto out;
1015                 r = 0;
1016                 break;
1017         }
1018         case KVM_TRANSLATE: {
1019                 struct kvm_translation tr;
1020
1021                 r = -EFAULT;
1022                 if (copy_from_user(&tr, argp, sizeof tr))
1023                         goto out;
1024                 r = kvm_vcpu_ioctl_translate(vcpu, &tr);
1025                 if (r)
1026                         goto out;
1027                 r = -EFAULT;
1028                 if (copy_to_user(argp, &tr, sizeof tr))
1029                         goto out;
1030                 r = 0;
1031                 break;
1032         }
1033         case KVM_INTERRUPT: {
1034                 struct kvm_interrupt irq;
1035
1036                 r = -EFAULT;
1037                 if (copy_from_user(&irq, argp, sizeof irq))
1038                         goto out;
1039                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1040                 if (r)
1041                         goto out;
1042                 r = 0;
1043                 break;
1044         }
1045         case KVM_DEBUG_GUEST: {
1046                 struct kvm_debug_guest dbg;
1047
1048                 r = -EFAULT;
1049                 if (copy_from_user(&dbg, argp, sizeof dbg))
1050                         goto out;
1051                 r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
1052                 if (r)
1053                         goto out;
1054                 r = 0;
1055                 break;
1056         }
1057         case KVM_SET_SIGNAL_MASK: {
1058                 struct kvm_signal_mask __user *sigmask_arg = argp;
1059                 struct kvm_signal_mask kvm_sigmask;
1060                 sigset_t sigset, *p;
1061
1062                 p = NULL;
1063                 if (argp) {
1064                         r = -EFAULT;
1065                         if (copy_from_user(&kvm_sigmask, argp,
1066                                            sizeof kvm_sigmask))
1067                                 goto out;
1068                         r = -EINVAL;
1069                         if (kvm_sigmask.len != sizeof sigset)
1070                                 goto out;
1071                         r = -EFAULT;
1072                         if (copy_from_user(&sigset, sigmask_arg->sigset,
1073                                            sizeof sigset))
1074                                 goto out;
1075                         p = &sigset;
1076                 }
1077                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1078                 break;
1079         }
1080         case KVM_GET_FPU: {
1081                 struct kvm_fpu fpu;
1082
1083                 memset(&fpu, 0, sizeof fpu);
1084                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
1085                 if (r)
1086                         goto out;
1087                 r = -EFAULT;
1088                 if (copy_to_user(argp, &fpu, sizeof fpu))
1089                         goto out;
1090                 r = 0;
1091                 break;
1092         }
1093         case KVM_SET_FPU: {
1094                 struct kvm_fpu fpu;
1095
1096                 r = -EFAULT;
1097                 if (copy_from_user(&fpu, argp, sizeof fpu))
1098                         goto out;
1099                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
1100                 if (r)
1101                         goto out;
1102                 r = 0;
1103                 break;
1104         }
1105         default:
1106                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1107         }
1108 out:
1109         return r;
1110 }
1111
1112 static long kvm_vm_ioctl(struct file *filp,
1113                            unsigned int ioctl, unsigned long arg)
1114 {
1115         struct kvm *kvm = filp->private_data;
1116         void __user *argp = (void __user *)arg;
1117         int r;
1118
1119         switch (ioctl) {
1120         case KVM_CREATE_VCPU:
1121                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1122                 if (r < 0)
1123                         goto out;
1124                 break;
1125         case KVM_SET_USER_MEMORY_REGION: {
1126                 struct kvm_userspace_memory_region kvm_userspace_mem;
1127
1128                 r = -EFAULT;
1129                 if (copy_from_user(&kvm_userspace_mem, argp,
1130                                                 sizeof kvm_userspace_mem))
1131                         goto out;
1132
1133                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
1134                 if (r)
1135                         goto out;
1136                 break;
1137         }
1138         case KVM_GET_DIRTY_LOG: {
1139                 struct kvm_dirty_log log;
1140
1141                 r = -EFAULT;
1142                 if (copy_from_user(&log, argp, sizeof log))
1143                         goto out;
1144                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
1145                 if (r)
1146                         goto out;
1147                 break;
1148         }
1149         default:
1150                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1151         }
1152 out:
1153         return r;
1154 }
1155
1156 static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
1157                                   unsigned long address,
1158                                   int *type)
1159 {
1160         struct kvm *kvm = vma->vm_file->private_data;
1161         unsigned long pgoff;
1162         struct page *page;
1163
1164         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1165         if (!kvm_is_visible_gfn(kvm, pgoff))
1166                 return NOPAGE_SIGBUS;
1167         /* current->mm->mmap_sem is already held so call lockless version */
1168         page = __gfn_to_page(kvm, pgoff);
1169         if (is_error_page(page)) {
1170                 kvm_release_page(page);
1171                 return NOPAGE_SIGBUS;
1172         }
1173         if (type != NULL)
1174                 *type = VM_FAULT_MINOR;
1175
1176         return page;
1177 }
1178
1179 static struct vm_operations_struct kvm_vm_vm_ops = {
1180         .nopage = kvm_vm_nopage,
1181 };
1182
1183 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
1184 {
1185         vma->vm_ops = &kvm_vm_vm_ops;
1186         return 0;
1187 }
1188
1189 static struct file_operations kvm_vm_fops = {
1190         .release        = kvm_vm_release,
1191         .unlocked_ioctl = kvm_vm_ioctl,
1192         .compat_ioctl   = kvm_vm_ioctl,
1193         .mmap           = kvm_vm_mmap,
1194 };
1195
1196 static int kvm_dev_ioctl_create_vm(void)
1197 {
1198         int fd, r;
1199         struct inode *inode;
1200         struct file *file;
1201         struct kvm *kvm;
1202
1203         kvm = kvm_create_vm();
1204         if (IS_ERR(kvm))
1205                 return PTR_ERR(kvm);
1206         r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
1207         if (r) {
1208                 kvm_destroy_vm(kvm);
1209                 return r;
1210         }
1211
1212         kvm->filp = file;
1213
1214         return fd;
1215 }
1216
1217 static long kvm_dev_ioctl(struct file *filp,
1218                           unsigned int ioctl, unsigned long arg)
1219 {
1220         void __user *argp = (void __user *)arg;
1221         long r = -EINVAL;
1222
1223         switch (ioctl) {
1224         case KVM_GET_API_VERSION:
1225                 r = -EINVAL;
1226                 if (arg)
1227                         goto out;
1228                 r = KVM_API_VERSION;
1229                 break;
1230         case KVM_CREATE_VM:
1231                 r = -EINVAL;
1232                 if (arg)
1233                         goto out;
1234                 r = kvm_dev_ioctl_create_vm();
1235                 break;
1236         case KVM_CHECK_EXTENSION: {
1237                 int ext = (long)argp;
1238
1239                 switch (ext) {
1240                 case KVM_CAP_IRQCHIP:
1241                 case KVM_CAP_HLT:
1242                 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
1243                 case KVM_CAP_USER_MEMORY:
1244                 case KVM_CAP_SET_TSS_ADDR:
1245                         r = 1;
1246                         break;
1247                 default:
1248                         r = 0;
1249                         break;
1250                 }
1251                 break;
1252         }
1253         case KVM_GET_VCPU_MMAP_SIZE:
1254                 r = -EINVAL;
1255                 if (arg)
1256                         goto out;
1257                 r = 2 * PAGE_SIZE;
1258                 break;
1259         default:
1260                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
1261         }
1262 out:
1263         return r;
1264 }
1265
1266 static struct file_operations kvm_chardev_ops = {
1267         .unlocked_ioctl = kvm_dev_ioctl,
1268         .compat_ioctl   = kvm_dev_ioctl,
1269 };
1270
1271 static struct miscdevice kvm_dev = {
1272         KVM_MINOR,
1273         "kvm",
1274         &kvm_chardev_ops,
1275 };
1276
1277 /*
1278  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
1279  * cached on it.
1280  */
1281 static void decache_vcpus_on_cpu(int cpu)
1282 {
1283         struct kvm *vm;
1284         struct kvm_vcpu *vcpu;
1285         int i;
1286
1287         spin_lock(&kvm_lock);
1288         list_for_each_entry(vm, &vm_list, vm_list)
1289                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
1290                         vcpu = vm->vcpus[i];
1291                         if (!vcpu)
1292                                 continue;
1293                         /*
1294                          * If the vcpu is locked, then it is running on some
1295                          * other cpu and therefore it is not cached on the
1296                          * cpu in question.
1297                          *
1298                          * If it's not locked, check the last cpu it executed
1299                          * on.
1300                          */
1301                         if (mutex_trylock(&vcpu->mutex)) {
1302                                 if (vcpu->cpu == cpu) {
1303                                         kvm_x86_ops->vcpu_decache(vcpu);
1304                                         vcpu->cpu = -1;
1305                                 }
1306                                 mutex_unlock(&vcpu->mutex);
1307                         }
1308                 }
1309         spin_unlock(&kvm_lock);
1310 }
1311
1312 static void hardware_enable(void *junk)
1313 {
1314         int cpu = raw_smp_processor_id();
1315
1316         if (cpu_isset(cpu, cpus_hardware_enabled))
1317                 return;
1318         cpu_set(cpu, cpus_hardware_enabled);
1319         kvm_x86_ops->hardware_enable(NULL);
1320 }
1321
1322 static void hardware_disable(void *junk)
1323 {
1324         int cpu = raw_smp_processor_id();
1325
1326         if (!cpu_isset(cpu, cpus_hardware_enabled))
1327                 return;
1328         cpu_clear(cpu, cpus_hardware_enabled);
1329         decache_vcpus_on_cpu(cpu);
1330         kvm_x86_ops->hardware_disable(NULL);
1331 }
1332
1333 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
1334                            void *v)
1335 {
1336         int cpu = (long)v;
1337
1338         val &= ~CPU_TASKS_FROZEN;
1339         switch (val) {
1340         case CPU_DYING:
1341                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1342                        cpu);
1343                 hardware_disable(NULL);
1344                 break;
1345         case CPU_UP_CANCELED:
1346                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1347                        cpu);
1348                 smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
1349                 break;
1350         case CPU_ONLINE:
1351                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
1352                        cpu);
1353                 smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
1354                 break;
1355         }
1356         return NOTIFY_OK;
1357 }
1358
1359 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
1360                       void *v)
1361 {
1362         if (val == SYS_RESTART) {
1363                 /*
1364                  * Some (well, at least mine) BIOSes hang on reboot if
1365                  * in vmx root mode.
1366                  */
1367                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1368                 on_each_cpu(hardware_disable, NULL, 0, 1);
1369         }
1370         return NOTIFY_OK;
1371 }
1372
1373 static struct notifier_block kvm_reboot_notifier = {
1374         .notifier_call = kvm_reboot,
1375         .priority = 0,
1376 };
1377
1378 void kvm_io_bus_init(struct kvm_io_bus *bus)
1379 {
1380         memset(bus, 0, sizeof(*bus));
1381 }
1382
1383 void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1384 {
1385         int i;
1386
1387         for (i = 0; i < bus->dev_count; i++) {
1388                 struct kvm_io_device *pos = bus->devs[i];
1389
1390                 kvm_iodevice_destructor(pos);
1391         }
1392 }
1393
1394 struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
1395 {
1396         int i;
1397
1398         for (i = 0; i < bus->dev_count; i++) {
1399                 struct kvm_io_device *pos = bus->devs[i];
1400
1401                 if (pos->in_range(pos, addr))
1402                         return pos;
1403         }
1404
1405         return NULL;
1406 }
1407
1408 void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
1409 {
1410         BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
1411
1412         bus->devs[bus->dev_count++] = dev;
1413 }
1414
1415 static struct notifier_block kvm_cpu_notifier = {
1416         .notifier_call = kvm_cpu_hotplug,
1417         .priority = 20, /* must be > scheduler priority */
1418 };
1419
1420 static u64 stat_get(void *_offset)
1421 {
1422         unsigned offset = (long)_offset;
1423         u64 total = 0;
1424         struct kvm *kvm;
1425         struct kvm_vcpu *vcpu;
1426         int i;
1427
1428         spin_lock(&kvm_lock);
1429         list_for_each_entry(kvm, &vm_list, vm_list)
1430                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
1431                         vcpu = kvm->vcpus[i];
1432                         if (vcpu)
1433                                 total += *(u32 *)((void *)vcpu + offset);
1434                 }
1435         spin_unlock(&kvm_lock);
1436         return total;
1437 }
1438
1439 DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
1440
1441 static __init void kvm_init_debug(void)
1442 {
1443         struct kvm_stats_debugfs_item *p;
1444
1445         debugfs_dir = debugfs_create_dir("kvm", NULL);
1446         for (p = debugfs_entries; p->name; ++p)
1447                 p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
1448                                                 (void *)(long)p->offset,
1449                                                 &stat_fops);
1450 }
1451
1452 static void kvm_exit_debug(void)
1453 {
1454         struct kvm_stats_debugfs_item *p;
1455
1456         for (p = debugfs_entries; p->name; ++p)
1457                 debugfs_remove(p->dentry);
1458         debugfs_remove(debugfs_dir);
1459 }
1460
1461 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
1462 {
1463         hardware_disable(NULL);
1464         return 0;
1465 }
1466
1467 static int kvm_resume(struct sys_device *dev)
1468 {
1469         hardware_enable(NULL);
1470         return 0;
1471 }
1472
1473 static struct sysdev_class kvm_sysdev_class = {
1474         .name = "kvm",
1475         .suspend = kvm_suspend,
1476         .resume = kvm_resume,
1477 };
1478
1479 static struct sys_device kvm_sysdev = {
1480         .id = 0,
1481         .cls = &kvm_sysdev_class,
1482 };
1483
1484 struct page *bad_page;
1485
1486 static inline
1487 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
1488 {
1489         return container_of(pn, struct kvm_vcpu, preempt_notifier);
1490 }
1491
1492 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
1493 {
1494         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1495
1496         kvm_x86_ops->vcpu_load(vcpu, cpu);
1497 }
1498
1499 static void kvm_sched_out(struct preempt_notifier *pn,
1500                           struct task_struct *next)
1501 {
1502         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1503
1504         kvm_x86_ops->vcpu_put(vcpu);
1505 }
1506
1507 int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
1508                   struct module *module)
1509 {
1510         int r;
1511         int cpu;
1512
1513         if (kvm_x86_ops) {
1514                 printk(KERN_ERR "kvm: already loaded the other module\n");
1515                 return -EEXIST;
1516         }
1517
1518         if (!ops->cpu_has_kvm_support()) {
1519                 printk(KERN_ERR "kvm: no hardware support\n");
1520                 return -EOPNOTSUPP;
1521         }
1522         if (ops->disabled_by_bios()) {
1523                 printk(KERN_ERR "kvm: disabled by bios\n");
1524                 return -EOPNOTSUPP;
1525         }
1526
1527         kvm_x86_ops = ops;
1528
1529         r = kvm_x86_ops->hardware_setup();
1530         if (r < 0)
1531                 goto out;
1532
1533         for_each_online_cpu(cpu) {
1534                 smp_call_function_single(cpu,
1535                                 kvm_x86_ops->check_processor_compatibility,
1536                                 &r, 0, 1);
1537                 if (r < 0)
1538                         goto out_free_0;
1539         }
1540
1541         on_each_cpu(hardware_enable, NULL, 0, 1);
1542         r = register_cpu_notifier(&kvm_cpu_notifier);
1543         if (r)
1544                 goto out_free_1;
1545         register_reboot_notifier(&kvm_reboot_notifier);
1546
1547         r = sysdev_class_register(&kvm_sysdev_class);
1548         if (r)
1549                 goto out_free_2;
1550
1551         r = sysdev_register(&kvm_sysdev);
1552         if (r)
1553                 goto out_free_3;
1554
1555         /* A kmem cache lets us meet the alignment requirements of fx_save. */
1556         kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
1557                                            __alignof__(struct kvm_vcpu), 0, 0);
1558         if (!kvm_vcpu_cache) {
1559                 r = -ENOMEM;
1560                 goto out_free_4;
1561         }
1562
1563         kvm_chardev_ops.owner = module;
1564
1565         r = misc_register(&kvm_dev);
1566         if (r) {
1567                 printk(KERN_ERR "kvm: misc device register failed\n");
1568                 goto out_free;
1569         }
1570
1571         kvm_preempt_ops.sched_in = kvm_sched_in;
1572         kvm_preempt_ops.sched_out = kvm_sched_out;
1573
1574         kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
1575
1576         return 0;
1577
1578 out_free:
1579         kmem_cache_destroy(kvm_vcpu_cache);
1580 out_free_4:
1581         sysdev_unregister(&kvm_sysdev);
1582 out_free_3:
1583         sysdev_class_unregister(&kvm_sysdev_class);
1584 out_free_2:
1585         unregister_reboot_notifier(&kvm_reboot_notifier);
1586         unregister_cpu_notifier(&kvm_cpu_notifier);
1587 out_free_1:
1588         on_each_cpu(hardware_disable, NULL, 0, 1);
1589 out_free_0:
1590         kvm_x86_ops->hardware_unsetup();
1591 out:
1592         kvm_x86_ops = NULL;
1593         return r;
1594 }
1595 EXPORT_SYMBOL_GPL(kvm_init_x86);
1596
1597 void kvm_exit_x86(void)
1598 {
1599         misc_deregister(&kvm_dev);
1600         kmem_cache_destroy(kvm_vcpu_cache);
1601         sysdev_unregister(&kvm_sysdev);
1602         sysdev_class_unregister(&kvm_sysdev_class);
1603         unregister_reboot_notifier(&kvm_reboot_notifier);
1604         unregister_cpu_notifier(&kvm_cpu_notifier);
1605         on_each_cpu(hardware_disable, NULL, 0, 1);
1606         kvm_x86_ops->hardware_unsetup();
1607         kvm_x86_ops = NULL;
1608 }
1609 EXPORT_SYMBOL_GPL(kvm_exit_x86);
1610
1611 static __init int kvm_init(void)
1612 {
1613         int r;
1614
1615         r = kvm_mmu_module_init();
1616         if (r)
1617                 goto out4;
1618
1619         kvm_init_debug();
1620
1621         kvm_arch_init();
1622
1623         bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1624
1625         if (bad_page == NULL) {
1626                 r = -ENOMEM;
1627                 goto out;
1628         }
1629
1630         return 0;
1631
1632 out:
1633         kvm_exit_debug();
1634         kvm_mmu_module_exit();
1635 out4:
1636         return r;
1637 }
1638
1639 static __exit void kvm_exit(void)
1640 {
1641         kvm_exit_debug();
1642         __free_page(bad_page);
1643         kvm_mmu_module_exit();
1644 }
1645
1646 module_init(kvm_init)
1647 module_exit(kvm_exit)