SAFE public projects git trees. - safe/jmp/linux-2.6/blob - virt/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "iodev.h"
  19
  20 #include <linux/kvm_host.h>
  21 #include <linux/kvm.h>
  22 #include <linux/module.h>
  23 #include <linux/errno.h>
  24 #include <linux/percpu.h>
  25 #include <linux/gfp.h>
  26 #include <linux/mm.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/vmalloc.h>
  29 #include <linux/reboot.h>
  30 #include <linux/debugfs.h>
  31 #include <linux/highmem.h>
  32 #include <linux/file.h>
  33 #include <linux/sysdev.h>
  34 #include <linux/cpu.h>
  35 #include <linux/sched.h>
  36 #include <linux/cpumask.h>
  37 #include <linux/smp.h>
  38 #include <linux/anon_inodes.h>
  39 #include <linux/profile.h>
  40 #include <linux/kvm_para.h>
  41 #include <linux/pagemap.h>
  42 #include <linux/mman.h>
  43 #include <linux/swap.h>
  44
  45 #include <asm/processor.h>
  46 #include <asm/io.h>
  47 #include <asm/uaccess.h>
  48 #include <asm/pgtable.h>
  49
  50 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
  51 #include "coalesced_mmio.h"
  52 #endif
  53
  54 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
  55 #include <linux/pci.h>
  56 #include <linux/interrupt.h>
  57 #include "irq.h"
  58 #endif
  59
  60 MODULE_AUTHOR("Qumranet");
  61 MODULE_LICENSE("GPL");
  62
  63 DEFINE_SPINLOCK(kvm_lock);
  64 LIST_HEAD(vm_list);
  65
  66 static cpumask_t cpus_hardware_enabled;
  67
  68 struct kmem_cache *kvm_vcpu_cache;
  69 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  70
  71 static __read_mostly struct preempt_ops kvm_preempt_ops;
  72
  73 struct dentry *kvm_debugfs_dir;
  74
  75 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  76                            unsigned long arg);
  77
  78 bool kvm_rebooting;
  79
  80 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
  81 static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
  82                                                       int assigned_dev_id)
  83 {
  84         struct list_head *ptr;
  85         struct kvm_assigned_dev_kernel *match;
  86
  87         list_for_each(ptr, head) {
  88                 match = list_entry(ptr, struct kvm_assigned_dev_kernel, list);
  89                 if (match->assigned_dev_id == assigned_dev_id)
  90                         return match;
  91         }
  92         return NULL;
  93 }
  94
  95 static void kvm_assigned_dev_interrupt_work_handler(struct work_struct *work)
  96 {
  97         struct kvm_assigned_dev_kernel *assigned_dev;
  98
  99         assigned_dev = container_of(work, struct kvm_assigned_dev_kernel,
 100                                     interrupt_work);
 101
 102         /* This is taken to safely inject irq inside the guest. When
 103          * the interrupt injection (or the ioapic code) uses a
 104          * finer-grained lock, update this
 105          */
 106         mutex_lock(&assigned_dev->kvm->lock);
 107         kvm_set_irq(assigned_dev->kvm,
 108                     assigned_dev->irq_source_id,
 109                     assigned_dev->guest_irq, 1);
 110         mutex_unlock(&assigned_dev->kvm->lock);
 111         kvm_put_kvm(assigned_dev->kvm);
 112 }
 113
 114 static irqreturn_t kvm_assigned_dev_intr(int irq, void *dev_id)
 115 {
 116         struct kvm_assigned_dev_kernel *assigned_dev =
 117                 (struct kvm_assigned_dev_kernel *) dev_id;
 118
 119         kvm_get_kvm(assigned_dev->kvm);
 120         schedule_work(&assigned_dev->interrupt_work);
 121         disable_irq_nosync(irq);
 122         return IRQ_HANDLED;
 123 }
 124
 125 /* Ack the irq line for an assigned device */
 126 static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier *kian)
 127 {
 128         struct kvm_assigned_dev_kernel *dev;
 129
 130         if (kian->gsi == -1)
 131                 return;
 132
 133         dev = container_of(kian, struct kvm_assigned_dev_kernel,
 134                            ack_notifier);
 135         kvm_set_irq(dev->kvm, dev->irq_source_id, dev->guest_irq, 0);
 136         enable_irq(dev->host_irq);
 137 }
 138
 139 static void kvm_free_assigned_device(struct kvm *kvm,
 140                                      struct kvm_assigned_dev_kernel
 141                                      *assigned_dev)
 142 {
 143         if (irqchip_in_kernel(kvm) && assigned_dev->irq_requested)
 144                 free_irq(assigned_dev->host_irq, (void *)assigned_dev);
 145
 146         kvm_unregister_irq_ack_notifier(&assigned_dev->ack_notifier);
 147         kvm_free_irq_source_id(kvm, assigned_dev->irq_source_id);
 148
 149         if (cancel_work_sync(&assigned_dev->interrupt_work))
 150                 /* We had pending work. That means we will have to take
 151                  * care of kvm_put_kvm.
 152                  */
 153                 kvm_put_kvm(kvm);
 154
 155         pci_reset_function(assigned_dev->dev);
 156
 157         pci_release_regions(assigned_dev->dev);
 158         pci_disable_device(assigned_dev->dev);
 159         pci_dev_put(assigned_dev->dev);
 160
 161         list_del(&assigned_dev->list);
 162         kfree(assigned_dev);
 163 }
 164
 165 void kvm_free_all_assigned_devices(struct kvm *kvm)
 166 {
 167         struct list_head *ptr, *ptr2;
 168         struct kvm_assigned_dev_kernel *assigned_dev;
 169
 170         list_for_each_safe(ptr, ptr2, &kvm->arch.assigned_dev_head) {
 171                 assigned_dev = list_entry(ptr,
 172                                           struct kvm_assigned_dev_kernel,
 173                                           list);
 174
 175                 kvm_free_assigned_device(kvm, assigned_dev);
 176         }
 177 }
 178
 179 static int assigned_device_update_intx(struct kvm *kvm,
 180                         struct kvm_assigned_dev_kernel *adev,
 181                         struct kvm_assigned_irq *airq)
 182 {
 183         if (adev->irq_requested) {
 184                 adev->guest_irq = airq->guest_irq;
 185                 adev->ack_notifier.gsi = airq->guest_irq;
 186                 return 0;
 187         }
 188
 189         if (irqchip_in_kernel(kvm)) {
 190                 if (!capable(CAP_SYS_RAWIO))
 191                         return -EPERM;
 192
 193                 if (airq->host_irq)
 194                         adev->host_irq = airq->host_irq;
 195                 else
 196                         adev->host_irq = adev->dev->irq;
 197                 adev->guest_irq = airq->guest_irq;
 198                 adev->ack_notifier.gsi = airq->guest_irq;
 199
 200                 /* Even though this is PCI, we don't want to use shared
 201                  * interrupts. Sharing host devices with guest-assigned devices
 202                  * on the same interrupt line is not a happy situation: there
 203                  * are going to be long delays in accepting, acking, etc.
 204                  */
 205                 if (request_irq(adev->host_irq, kvm_assigned_dev_intr,
 206                                 0, "kvm_assigned_intx_device", (void *)adev))
 207                         return -EIO;
 208         }
 209
 210         adev->irq_requested = true;
 211         return 0;
 212 }
 213
 214 static int kvm_vm_ioctl_assign_irq(struct kvm *kvm,
 215                                    struct kvm_assigned_irq
 216                                    *assigned_irq)
 217 {
 218         int r = 0;
 219         struct kvm_assigned_dev_kernel *match;
 220
 221         mutex_lock(&kvm->lock);
 222
 223         match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 224                                       assigned_irq->assigned_dev_id);
 225         if (!match) {
 226                 mutex_unlock(&kvm->lock);
 227                 return -EINVAL;
 228         }
 229
 230         if (!match->irq_requested) {
 231                 INIT_WORK(&match->interrupt_work,
 232                                 kvm_assigned_dev_interrupt_work_handler);
 233                 if (irqchip_in_kernel(kvm)) {
 234                         /* Register ack nofitier */
 235                         match->ack_notifier.gsi = -1;
 236                         match->ack_notifier.irq_acked =
 237                                         kvm_assigned_dev_ack_irq;
 238                         kvm_register_irq_ack_notifier(kvm,
 239                                         &match->ack_notifier);
 240
 241                         /* Request IRQ source ID */
 242                         r = kvm_request_irq_source_id(kvm);
 243                         if (r < 0)
 244                                 goto out_release;
 245                         else
 246                                 match->irq_source_id = r;
 247                 }
 248         }
 249
 250         r = assigned_device_update_intx(kvm, match, assigned_irq);
 251         if (r)
 252                 goto out_release;
 253
 254         mutex_unlock(&kvm->lock);
 255         return r;
 256 out_release:
 257         mutex_unlock(&kvm->lock);
 258         kvm_free_assigned_device(kvm, match);
 259         return r;
 260 }
 261
 262 static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
 263                                       struct kvm_assigned_pci_dev *assigned_dev)
 264 {
 265         int r = 0;
 266         struct kvm_assigned_dev_kernel *match;
 267         struct pci_dev *dev;
 268
 269         mutex_lock(&kvm->lock);
 270
 271         match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 272                                       assigned_dev->assigned_dev_id);
 273         if (match) {
 274                 /* device already assigned */
 275                 r = -EINVAL;
 276                 goto out;
 277         }
 278
 279         match = kzalloc(sizeof(struct kvm_assigned_dev_kernel), GFP_KERNEL);
 280         if (match == NULL) {
 281                 printk(KERN_INFO "%s: Couldn't allocate memory\n",
 282                        __func__);
 283                 r = -ENOMEM;
 284                 goto out;
 285         }
 286         dev = pci_get_bus_and_slot(assigned_dev->busnr,
 287                                    assigned_dev->devfn);
 288         if (!dev) {
 289                 printk(KERN_INFO "%s: host device not found\n", __func__);
 290                 r = -EINVAL;
 291                 goto out_free;
 292         }
 293         if (pci_enable_device(dev)) {
 294                 printk(KERN_INFO "%s: Could not enable PCI device\n", __func__);
 295                 r = -EBUSY;
 296                 goto out_put;
 297         }
 298         r = pci_request_regions(dev, "kvm_assigned_device");
 299         if (r) {
 300                 printk(KERN_INFO "%s: Could not get access to device regions\n",
 301                        __func__);
 302                 goto out_disable;
 303         }
 304
 305         pci_reset_function(dev);
 306
 307         match->assigned_dev_id = assigned_dev->assigned_dev_id;
 308         match->host_busnr = assigned_dev->busnr;
 309         match->host_devfn = assigned_dev->devfn;
 310         match->dev = dev;
 311
 312         match->kvm = kvm;
 313
 314         list_add(&match->list, &kvm->arch.assigned_dev_head);
 315
 316         if (assigned_dev->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU) {
 317                 r = kvm_iommu_map_guest(kvm, match);
 318                 if (r)
 319                         goto out_list_del;
 320         }
 321
 322 out:
 323         mutex_unlock(&kvm->lock);
 324         return r;
 325 out_list_del:
 326         list_del(&match->list);
 327         pci_release_regions(dev);
 328 out_disable:
 329         pci_disable_device(dev);
 330 out_put:
 331         pci_dev_put(dev);
 332 out_free:
 333         kfree(match);
 334         mutex_unlock(&kvm->lock);
 335         return r;
 336 }
 337 #endif
 338
 339 static inline int valid_vcpu(int n)
 340 {
 341         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 342 }
 343
 344 inline int kvm_is_mmio_pfn(pfn_t pfn)
 345 {
 346         if (pfn_valid(pfn))
 347                 return PageReserved(pfn_to_page(pfn));
 348
 349         return true;
 350 }
 351
 352 /*
 353  * Switches to specified vcpu, until a matching vcpu_put()
 354  */
 355 void vcpu_load(struct kvm_vcpu *vcpu)
 356 {
 357         int cpu;
 358
 359         mutex_lock(&vcpu->mutex);
 360         cpu = get_cpu();
 361         preempt_notifier_register(&vcpu->preempt_notifier);
 362         kvm_arch_vcpu_load(vcpu, cpu);
 363         put_cpu();
 364 }
 365
 366 void vcpu_put(struct kvm_vcpu *vcpu)
 367 {
 368         preempt_disable();
 369         kvm_arch_vcpu_put(vcpu);
 370         preempt_notifier_unregister(&vcpu->preempt_notifier);
 371         preempt_enable();
 372         mutex_unlock(&vcpu->mutex);
 373 }
 374
 375 static void ack_flush(void *_completed)
 376 {
 377 }
 378
 379 void kvm_flush_remote_tlbs(struct kvm *kvm)
 380 {
 381         int i, cpu, me;
 382         cpumask_t cpus;
 383         struct kvm_vcpu *vcpu;
 384
 385         me = get_cpu();
 386         cpus_clear(cpus);
 387         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 388                 vcpu = kvm->vcpus[i];
 389                 if (!vcpu)
 390                         continue;
 391                 if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
 392                         continue;
 393                 cpu = vcpu->cpu;
 394                 if (cpu != -1 && cpu != me)
 395                         cpu_set(cpu, cpus);
 396         }
 397         if (cpus_empty(cpus))
 398                 goto out;
 399         ++kvm->stat.remote_tlb_flush;
 400         smp_call_function_mask(cpus, ack_flush, NULL, 1);
 401 out:
 402         put_cpu();
 403 }
 404
 405 void kvm_reload_remote_mmus(struct kvm *kvm)
 406 {
 407         int i, cpu, me;
 408         cpumask_t cpus;
 409         struct kvm_vcpu *vcpu;
 410
 411         me = get_cpu();
 412         cpus_clear(cpus);
 413         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 414                 vcpu = kvm->vcpus[i];
 415                 if (!vcpu)
 416                         continue;
 417                 if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
 418                         continue;
 419                 cpu = vcpu->cpu;
 420                 if (cpu != -1 && cpu != me)
 421                         cpu_set(cpu, cpus);
 422         }
 423         if (cpus_empty(cpus))
 424                 goto out;
 425         smp_call_function_mask(cpus, ack_flush, NULL, 1);
 426 out:
 427         put_cpu();
 428 }
 429
 430
 431 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 432 {
 433         struct page *page;
 434         int r;
 435
 436         mutex_init(&vcpu->mutex);
 437         vcpu->cpu = -1;
 438         vcpu->kvm = kvm;
 439         vcpu->vcpu_id = id;
 440         init_waitqueue_head(&vcpu->wq);
 441
 442         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 443         if (!page) {
 444                 r = -ENOMEM;
 445                 goto fail;
 446         }
 447         vcpu->run = page_address(page);
 448
 449         r = kvm_arch_vcpu_init(vcpu);
 450         if (r < 0)
 451                 goto fail_free_run;
 452         return 0;
 453
 454 fail_free_run:
 455         free_page((unsigned long)vcpu->run);
 456 fail:
 457         return r;
 458 }
 459 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 460
 461 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 462 {
 463         kvm_arch_vcpu_uninit(vcpu);
 464         free_page((unsigned long)vcpu->run);
 465 }
 466 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 467
 468 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 469 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 470 {
 471         return container_of(mn, struct kvm, mmu_notifier);
 472 }
 473
 474 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 475                                              struct mm_struct *mm,
 476                                              unsigned long address)
 477 {
 478         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 479         int need_tlb_flush;
 480
 481         /*
 482          * When ->invalidate_page runs, the linux pte has been zapped
 483          * already but the page is still allocated until
 484          * ->invalidate_page returns. So if we increase the sequence
 485          * here the kvm page fault will notice if the spte can't be
 486          * established because the page is going to be freed. If
 487          * instead the kvm page fault establishes the spte before
 488          * ->invalidate_page runs, kvm_unmap_hva will release it
 489          * before returning.
 490          *
 491          * The sequence increase only need to be seen at spin_unlock
 492          * time, and not at spin_lock time.
 493          *
 494          * Increasing the sequence after the spin_unlock would be
 495          * unsafe because the kvm page fault could then establish the
 496          * pte after kvm_unmap_hva returned, without noticing the page
 497          * is going to be freed.
 498          */
 499         spin_lock(&kvm->mmu_lock);
 500         kvm->mmu_notifier_seq++;
 501         need_tlb_flush = kvm_unmap_hva(kvm, address);
 502         spin_unlock(&kvm->mmu_lock);
 503
 504         /* we've to flush the tlb before the pages can be freed */
 505         if (need_tlb_flush)
 506                 kvm_flush_remote_tlbs(kvm);
 507
 508 }
 509
 510 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 511                                                     struct mm_struct *mm,
 512                                                     unsigned long start,
 513                                                     unsigned long end)
 514 {
 515         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 516         int need_tlb_flush = 0;
 517
 518         spin_lock(&kvm->mmu_lock);
 519         /*
 520          * The count increase must become visible at unlock time as no
 521          * spte can be established without taking the mmu_lock and
 522          * count is also read inside the mmu_lock critical section.
 523          */
 524         kvm->mmu_notifier_count++;
 525         for (; start < end; start += PAGE_SIZE)
 526                 need_tlb_flush |= kvm_unmap_hva(kvm, start);
 527         spin_unlock(&kvm->mmu_lock);
 528
 529         /* we've to flush the tlb before the pages can be freed */
 530         if (need_tlb_flush)
 531                 kvm_flush_remote_tlbs(kvm);
 532 }
 533
 534 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 535                                                   struct mm_struct *mm,
 536                                                   unsigned long start,
 537                                                   unsigned long end)
 538 {
 539         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 540
 541         spin_lock(&kvm->mmu_lock);
 542         /*
 543          * This sequence increase will notify the kvm page fault that
 544          * the page that is going to be mapped in the spte could have
 545          * been freed.
 546          */
 547         kvm->mmu_notifier_seq++;
 548         /*
 549          * The above sequence increase must be visible before the
 550          * below count decrease but both values are read by the kvm
 551          * page fault under mmu_lock spinlock so we don't need to add
 552          * a smb_wmb() here in between the two.
 553          */
 554         kvm->mmu_notifier_count--;
 555         spin_unlock(&kvm->mmu_lock);
 556
 557         BUG_ON(kvm->mmu_notifier_count < 0);
 558 }
 559
 560 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
 561                                               struct mm_struct *mm,
 562                                               unsigned long address)
 563 {
 564         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 565         int young;
 566
 567         spin_lock(&kvm->mmu_lock);
 568         young = kvm_age_hva(kvm, address);
 569         spin_unlock(&kvm->mmu_lock);
 570
 571         if (young)
 572                 kvm_flush_remote_tlbs(kvm);
 573
 574         return young;
 575 }
 576
 577 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 578         .invalidate_page        = kvm_mmu_notifier_invalidate_page,
 579         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
 580         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
 581         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
 582 };
 583 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
 584
 585 static struct kvm *kvm_create_vm(void)
 586 {
 587         struct kvm *kvm = kvm_arch_create_vm();
 588 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 589         struct page *page;
 590 #endif
 591
 592         if (IS_ERR(kvm))
 593                 goto out;
 594
 595 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 596         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 597         if (!page) {
 598                 kfree(kvm);
 599                 return ERR_PTR(-ENOMEM);
 600         }
 601         kvm->coalesced_mmio_ring =
 602                         (struct kvm_coalesced_mmio_ring *)page_address(page);
 603 #endif
 604
 605 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 606         {
 607                 int err;
 608                 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
 609                 err = mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 610                 if (err) {
 611 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 612                         put_page(page);
 613 #endif
 614                         kfree(kvm);
 615                         return ERR_PTR(err);
 616                 }
 617         }
 618 #endif
 619
 620         kvm->mm = current->mm;
 621         atomic_inc(&kvm->mm->mm_count);
 622         spin_lock_init(&kvm->mmu_lock);
 623         kvm_io_bus_init(&kvm->pio_bus);
 624         mutex_init(&kvm->lock);
 625         kvm_io_bus_init(&kvm->mmio_bus);
 626         init_rwsem(&kvm->slots_lock);
 627         atomic_set(&kvm->users_count, 1);
 628         spin_lock(&kvm_lock);
 629         list_add(&kvm->vm_list, &vm_list);
 630         spin_unlock(&kvm_lock);
 631 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 632         kvm_coalesced_mmio_init(kvm);
 633 #endif
 634 out:
 635         return kvm;
 636 }
 637
 638 /*
 639  * Free any memory in @free but not in @dont.
 640  */
 641 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 642                                   struct kvm_memory_slot *dont)
 643 {
 644         if (!dont || free->rmap != dont->rmap)
 645                 vfree(free->rmap);
 646
 647         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 648                 vfree(free->dirty_bitmap);
 649
 650         if (!dont || free->lpage_info != dont->lpage_info)
 651                 vfree(free->lpage_info);
 652
 653         free->npages = 0;
 654         free->dirty_bitmap = NULL;
 655         free->rmap = NULL;
 656         free->lpage_info = NULL;
 657 }
 658
 659 void kvm_free_physmem(struct kvm *kvm)
 660 {
 661         int i;
 662
 663         for (i = 0; i < kvm->nmemslots; ++i)
 664                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 665 }
 666
 667 static void kvm_destroy_vm(struct kvm *kvm)
 668 {
 669         struct mm_struct *mm = kvm->mm;
 670
 671         spin_lock(&kvm_lock);
 672         list_del(&kvm->vm_list);
 673         spin_unlock(&kvm_lock);
 674         kvm_io_bus_destroy(&kvm->pio_bus);
 675         kvm_io_bus_destroy(&kvm->mmio_bus);
 676 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 677         if (kvm->coalesced_mmio_ring != NULL)
 678                 free_page((unsigned long)kvm->coalesced_mmio_ring);
 679 #endif
 680 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 681         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 682 #endif
 683         kvm_arch_destroy_vm(kvm);
 684         mmdrop(mm);
 685 }
 686
 687 void kvm_get_kvm(struct kvm *kvm)
 688 {
 689         atomic_inc(&kvm->users_count);
 690 }
 691 EXPORT_SYMBOL_GPL(kvm_get_kvm);
 692
 693 void kvm_put_kvm(struct kvm *kvm)
 694 {
 695         if (atomic_dec_and_test(&kvm->users_count))
 696                 kvm_destroy_vm(kvm);
 697 }
 698 EXPORT_SYMBOL_GPL(kvm_put_kvm);
 699
 700
 701 static int kvm_vm_release(struct inode *inode, struct file *filp)
 702 {
 703         struct kvm *kvm = filp->private_data;
 704
 705         kvm_put_kvm(kvm);
 706         return 0;
 707 }
 708
 709 /*
 710  * Allocate some memory and give it an address in the guest physical address
 711  * space.
 712  *
 713  * Discontiguous memory is allowed, mostly for framebuffers.
 714  *
 715  * Must be called holding mmap_sem for write.
 716  */
 717 int __kvm_set_memory_region(struct kvm *kvm,
 718                             struct kvm_userspace_memory_region *mem,
 719                             int user_alloc)
 720 {
 721         int r;
 722         gfn_t base_gfn;
 723         unsigned long npages;
 724         unsigned long i;
 725         struct kvm_memory_slot *memslot;
 726         struct kvm_memory_slot old, new;
 727
 728         r = -EINVAL;
 729         /* General sanity checks */
 730         if (mem->memory_size & (PAGE_SIZE - 1))
 731                 goto out;
 732         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 733                 goto out;
 734         if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
 735                 goto out;
 736         if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 737                 goto out;
 738         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 739                 goto out;
 740
 741         memslot = &kvm->memslots[mem->slot];
 742         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 743         npages = mem->memory_size >> PAGE_SHIFT;
 744
 745         if (!npages)
 746                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 747
 748         new = old = *memslot;
 749
 750         new.base_gfn = base_gfn;
 751         new.npages = npages;
 752         new.flags = mem->flags;
 753
 754         /* Disallow changing a memory slot's size. */
 755         r = -EINVAL;
 756         if (npages && old.npages && npages != old.npages)
 757                 goto out_free;
 758
 759         /* Check for overlaps */
 760         r = -EEXIST;
 761         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 762                 struct kvm_memory_slot *s = &kvm->memslots[i];
 763
 764                 if (s == memslot)
 765                         continue;
 766                 if (!((base_gfn + npages <= s->base_gfn) ||
 767                       (base_gfn >= s->base_gfn + s->npages)))
 768                         goto out_free;
 769         }
 770
 771         /* Free page dirty bitmap if unneeded */
 772         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 773                 new.dirty_bitmap = NULL;
 774
 775         r = -ENOMEM;
 776
 777         /* Allocate if a slot is being created */
 778 #ifndef CONFIG_S390
 779         if (npages && !new.rmap) {
 780                 new.rmap = vmalloc(npages * sizeof(struct page *));
 781
 782                 if (!new.rmap)
 783                         goto out_free;
 784
 785                 memset(new.rmap, 0, npages * sizeof(*new.rmap));
 786
 787                 new.user_alloc = user_alloc;
 788                 /*
 789                  * hva_to_rmmap() serialzies with the mmu_lock and to be
 790                  * safe it has to ignore memslots with !user_alloc &&
 791                  * !userspace_addr.
 792                  */
 793                 if (user_alloc)
 794                         new.userspace_addr = mem->userspace_addr;
 795                 else
 796                         new.userspace_addr = 0;
 797         }
 798         if (npages && !new.lpage_info) {
 799                 int largepages = npages / KVM_PAGES_PER_HPAGE;
 800                 if (npages % KVM_PAGES_PER_HPAGE)
 801                         largepages++;
 802                 if (base_gfn % KVM_PAGES_PER_HPAGE)
 803                         largepages++;
 804
 805                 new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));
 806
 807                 if (!new.lpage_info)
 808                         goto out_free;
 809
 810                 memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));
 811
 812                 if (base_gfn % KVM_PAGES_PER_HPAGE)
 813                         new.lpage_info[0].write_count = 1;
 814                 if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
 815                         new.lpage_info[largepages-1].write_count = 1;
 816         }
 817
 818         /* Allocate page dirty bitmap if needed */
 819         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 820                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 821
 822                 new.dirty_bitmap = vmalloc(dirty_bytes);
 823                 if (!new.dirty_bitmap)
 824                         goto out_free;
 825                 memset(new.dirty_bitmap, 0, dirty_bytes);
 826         }
 827 #endif /* not defined CONFIG_S390 */
 828
 829         if (!npages)
 830                 kvm_arch_flush_shadow(kvm);
 831
 832         spin_lock(&kvm->mmu_lock);
 833         if (mem->slot >= kvm->nmemslots)
 834                 kvm->nmemslots = mem->slot + 1;
 835
 836         *memslot = new;
 837         spin_unlock(&kvm->mmu_lock);
 838
 839         r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
 840         if (r) {
 841                 spin_lock(&kvm->mmu_lock);
 842                 *memslot = old;
 843                 spin_unlock(&kvm->mmu_lock);
 844                 goto out_free;
 845         }
 846
 847         kvm_free_physmem_slot(&old, &new);
 848 #ifdef CONFIG_DMAR
 849         /* map the pages in iommu page table */
 850         r = kvm_iommu_map_pages(kvm, base_gfn, npages);
 851         if (r)
 852                 goto out;
 853 #endif
 854         return 0;
 855
 856 out_free:
 857         kvm_free_physmem_slot(&new, &old);
 858 out:
 859         return r;
 860
 861 }
 862 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 863
 864 int kvm_set_memory_region(struct kvm *kvm,
 865                           struct kvm_userspace_memory_region *mem,
 866                           int user_alloc)
 867 {
 868         int r;
 869
 870         down_write(&kvm->slots_lock);
 871         r = __kvm_set_memory_region(kvm, mem, user_alloc);
 872         up_write(&kvm->slots_lock);
 873         return r;
 874 }
 875 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
 876
 877 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 878                                    struct
 879                                    kvm_userspace_memory_region *mem,
 880                                    int user_alloc)
 881 {
 882         if (mem->slot >= KVM_MEMORY_SLOTS)
 883                 return -EINVAL;
 884         return kvm_set_memory_region(kvm, mem, user_alloc);
 885 }
 886
 887 int kvm_get_dirty_log(struct kvm *kvm,
 888                         struct kvm_dirty_log *log, int *is_dirty)
 889 {
 890         struct kvm_memory_slot *memslot;
 891         int r, i;
 892         int n;
 893         unsigned long any = 0;
 894
 895         r = -EINVAL;
 896         if (log->slot >= KVM_MEMORY_SLOTS)
 897                 goto out;
 898
 899         memslot = &kvm->memslots[log->slot];
 900         r = -ENOENT;
 901         if (!memslot->dirty_bitmap)
 902                 goto out;
 903
 904         n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 905
 906         for (i = 0; !any && i < n/sizeof(long); ++i)
 907                 any = memslot->dirty_bitmap[i];
 908
 909         r = -EFAULT;
 910         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 911                 goto out;
 912
 913         if (any)
 914                 *is_dirty = 1;
 915
 916         r = 0;
 917 out:
 918         return r;
 919 }
 920
 921 int is_error_page(struct page *page)
 922 {
 923         return page == bad_page;
 924 }
 925 EXPORT_SYMBOL_GPL(is_error_page);
 926
 927 int is_error_pfn(pfn_t pfn)
 928 {
 929         return pfn == bad_pfn;
 930 }
 931 EXPORT_SYMBOL_GPL(is_error_pfn);
 932
 933 static inline unsigned long bad_hva(void)
 934 {
 935         return PAGE_OFFSET;
 936 }
 937
 938 int kvm_is_error_hva(unsigned long addr)
 939 {
 940         return addr == bad_hva();
 941 }
 942 EXPORT_SYMBOL_GPL(kvm_is_error_hva);
 943
 944 struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
 945 {
 946         int i;
 947
 948         for (i = 0; i < kvm->nmemslots; ++i) {
 949                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 950
 951                 if (gfn >= memslot->base_gfn
 952                     && gfn < memslot->base_gfn + memslot->npages)
 953                         return memslot;
 954         }
 955         return NULL;
 956 }
 957 EXPORT_SYMBOL_GPL(gfn_to_memslot_unaliased);
 958
 959 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 960 {
 961         gfn = unalias_gfn(kvm, gfn);
 962         return gfn_to_memslot_unaliased(kvm, gfn);
 963 }
 964
 965 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
 966 {
 967         int i;
 968
 969         gfn = unalias_gfn(kvm, gfn);
 970         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 971                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 972
 973                 if (gfn >= memslot->base_gfn
 974                     && gfn < memslot->base_gfn + memslot->npages)
 975                         return 1;
 976         }
 977         return 0;
 978 }
 979 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
 980
 981 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
 982 {
 983         struct kvm_memory_slot *slot;
 984
 985         gfn = unalias_gfn(kvm, gfn);
 986         slot = gfn_to_memslot_unaliased(kvm, gfn);
 987         if (!slot)
 988                 return bad_hva();
 989         return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
 990 }
 991 EXPORT_SYMBOL_GPL(gfn_to_hva);
 992
 993 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
 994 {
 995         struct page *page[1];
 996         unsigned long addr;
 997         int npages;
 998         pfn_t pfn;
 999
1000         might_sleep();
1001
1002         addr = gfn_to_hva(kvm, gfn);
1003         if (kvm_is_error_hva(addr)) {
1004                 get_page(bad_page);
1005                 return page_to_pfn(bad_page);
1006         }
1007
1008         npages = get_user_pages_fast(addr, 1, 1, page);
1009
1010         if (unlikely(npages != 1)) {
1011                 struct vm_area_struct *vma;
1012
1013                 down_read(&current->mm->mmap_sem);
1014                 vma = find_vma(current->mm, addr);
1015
1016                 if (vma == NULL || addr < vma->vm_start ||
1017                     !(vma->vm_flags & VM_PFNMAP)) {
1018                         up_read(&current->mm->mmap_sem);
1019                         get_page(bad_page);
1020                         return page_to_pfn(bad_page);
1021                 }
1022
1023                 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1024                 up_read(&current->mm->mmap_sem);
1025                 BUG_ON(!kvm_is_mmio_pfn(pfn));
1026         } else
1027                 pfn = page_to_pfn(page[0]);
1028
1029         return pfn;
1030 }
1031
1032 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1033
1034 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1035 {
1036         pfn_t pfn;
1037
1038         pfn = gfn_to_pfn(kvm, gfn);
1039         if (!kvm_is_mmio_pfn(pfn))
1040                 return pfn_to_page(pfn);
1041
1042         WARN_ON(kvm_is_mmio_pfn(pfn));
1043
1044         get_page(bad_page);
1045         return bad_page;
1046 }
1047
1048 EXPORT_SYMBOL_GPL(gfn_to_page);
1049
1050 void kvm_release_page_clean(struct page *page)
1051 {
1052         kvm_release_pfn_clean(page_to_pfn(page));
1053 }
1054 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1055
1056 void kvm_release_pfn_clean(pfn_t pfn)
1057 {
1058         if (!kvm_is_mmio_pfn(pfn))
1059                 put_page(pfn_to_page(pfn));
1060 }
1061 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1062
1063 void kvm_release_page_dirty(struct page *page)
1064 {
1065         kvm_release_pfn_dirty(page_to_pfn(page));
1066 }
1067 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1068
1069 void kvm_release_pfn_dirty(pfn_t pfn)
1070 {
1071         kvm_set_pfn_dirty(pfn);
1072         kvm_release_pfn_clean(pfn);
1073 }
1074 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1075
1076 void kvm_set_page_dirty(struct page *page)
1077 {
1078         kvm_set_pfn_dirty(page_to_pfn(page));
1079 }
1080 EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
1081
1082 void kvm_set_pfn_dirty(pfn_t pfn)
1083 {
1084         if (!kvm_is_mmio_pfn(pfn)) {
1085                 struct page *page = pfn_to_page(pfn);
1086                 if (!PageReserved(page))
1087                         SetPageDirty(page);
1088         }
1089 }
1090 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1091
1092 void kvm_set_pfn_accessed(pfn_t pfn)
1093 {
1094         if (!kvm_is_mmio_pfn(pfn))
1095                 mark_page_accessed(pfn_to_page(pfn));
1096 }
1097 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1098
1099 void kvm_get_pfn(pfn_t pfn)
1100 {
1101         if (!kvm_is_mmio_pfn(pfn))
1102                 get_page(pfn_to_page(pfn));
1103 }
1104 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1105
1106 static int next_segment(unsigned long len, int offset)
1107 {
1108         if (len > PAGE_SIZE - offset)
1109                 return PAGE_SIZE - offset;
1110         else
1111                 return len;
1112 }
1113
1114 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1115                         int len)
1116 {
1117         int r;
1118         unsigned long addr;
1119
1120         addr = gfn_to_hva(kvm, gfn);
1121         if (kvm_is_error_hva(addr))
1122                 return -EFAULT;
1123         r = copy_from_user(data, (void __user *)addr + offset, len);
1124         if (r)
1125                 return -EFAULT;
1126         return 0;
1127 }
1128 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1129
1130 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1131 {
1132         gfn_t gfn = gpa >> PAGE_SHIFT;
1133         int seg;
1134         int offset = offset_in_page(gpa);
1135         int ret;
1136
1137         while ((seg = next_segment(len, offset)) != 0) {
1138                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1139                 if (ret < 0)
1140                         return ret;
1141                 offset = 0;
1142                 len -= seg;
1143                 data += seg;
1144                 ++gfn;
1145         }
1146         return 0;
1147 }
1148 EXPORT_SYMBOL_GPL(kvm_read_guest);
1149
1150 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1151                           unsigned long len)
1152 {
1153         int r;
1154         unsigned long addr;
1155         gfn_t gfn = gpa >> PAGE_SHIFT;
1156         int offset = offset_in_page(gpa);
1157
1158         addr = gfn_to_hva(kvm, gfn);
1159         if (kvm_is_error_hva(addr))
1160                 return -EFAULT;
1161         pagefault_disable();
1162         r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1163         pagefault_enable();
1164         if (r)
1165                 return -EFAULT;
1166         return 0;
1167 }
1168 EXPORT_SYMBOL(kvm_read_guest_atomic);
1169
1170 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1171                          int offset, int len)
1172 {
1173         int r;
1174         unsigned long addr;
1175
1176         addr = gfn_to_hva(kvm, gfn);
1177         if (kvm_is_error_hva(addr))
1178                 return -EFAULT;
1179         r = copy_to_user((void __user *)addr + offset, data, len);
1180         if (r)
1181                 return -EFAULT;
1182         mark_page_dirty(kvm, gfn);
1183         return 0;
1184 }
1185 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1186
1187 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1188                     unsigned long len)
1189 {
1190         gfn_t gfn = gpa >> PAGE_SHIFT;
1191         int seg;
1192         int offset = offset_in_page(gpa);
1193         int ret;
1194
1195         while ((seg = next_segment(len, offset)) != 0) {
1196                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1197                 if (ret < 0)
1198                         return ret;
1199                 offset = 0;
1200                 len -= seg;
1201                 data += seg;
1202                 ++gfn;
1203         }
1204         return 0;
1205 }
1206
1207 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1208 {
1209         return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
1210 }
1211 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1212
1213 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1214 {
1215         gfn_t gfn = gpa >> PAGE_SHIFT;
1216         int seg;
1217         int offset = offset_in_page(gpa);
1218         int ret;
1219
1220         while ((seg = next_segment(len, offset)) != 0) {
1221                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1222                 if (ret < 0)
1223                         return ret;
1224                 offset = 0;
1225                 len -= seg;
1226                 ++gfn;
1227         }
1228         return 0;
1229 }
1230 EXPORT_SYMBOL_GPL(kvm_clear_guest);
1231
1232 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1233 {
1234         struct kvm_memory_slot *memslot;
1235
1236         gfn = unalias_gfn(kvm, gfn);
1237         memslot = gfn_to_memslot_unaliased(kvm, gfn);
1238         if (memslot && memslot->dirty_bitmap) {
1239                 unsigned long rel_gfn = gfn - memslot->base_gfn;
1240
1241                 /* avoid RMW */
1242                 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
1243                         set_bit(rel_gfn, memslot->dirty_bitmap);
1244         }
1245 }
1246
1247 /*
1248  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1249  */
1250 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1251 {
1252         DEFINE_WAIT(wait);
1253
1254         for (;;) {
1255                 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1256
1257                 if (kvm_cpu_has_interrupt(vcpu) ||
1258                     kvm_cpu_has_pending_timer(vcpu) ||
1259                     kvm_arch_vcpu_runnable(vcpu)) {
1260                         set_bit(KVM_REQ_UNHALT, &vcpu->requests);
1261                         break;
1262                 }
1263                 if (signal_pending(current))
1264                         break;
1265
1266                 vcpu_put(vcpu);
1267                 schedule();
1268                 vcpu_load(vcpu);
1269         }
1270
1271         finish_wait(&vcpu->wq, &wait);
1272 }
1273
1274 void kvm_resched(struct kvm_vcpu *vcpu)
1275 {
1276         if (!need_resched())
1277                 return;
1278         cond_resched();
1279 }
1280 EXPORT_SYMBOL_GPL(kvm_resched);
1281
1282 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1283 {
1284         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1285         struct page *page;
1286
1287         if (vmf->pgoff == 0)
1288                 page = virt_to_page(vcpu->run);
1289 #ifdef CONFIG_X86
1290         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1291                 page = virt_to_page(vcpu->arch.pio_data);
1292 #endif
1293 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1294         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1295                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1296 #endif
1297         else
1298                 return VM_FAULT_SIGBUS;
1299         get_page(page);
1300         vmf->page = page;
1301         return 0;
1302 }
1303
1304 static struct vm_operations_struct kvm_vcpu_vm_ops = {
1305         .fault = kvm_vcpu_fault,
1306 };
1307
1308 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1309 {
1310         vma->vm_ops = &kvm_vcpu_vm_ops;
1311         return 0;
1312 }
1313
1314 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1315 {
1316         struct kvm_vcpu *vcpu = filp->private_data;
1317
1318         kvm_put_kvm(vcpu->kvm);
1319         return 0;
1320 }
1321
1322 static const struct file_operations kvm_vcpu_fops = {
1323         .release        = kvm_vcpu_release,
1324         .unlocked_ioctl = kvm_vcpu_ioctl,
1325         .compat_ioctl   = kvm_vcpu_ioctl,
1326         .mmap           = kvm_vcpu_mmap,
1327 };
1328
1329 /*
1330  * Allocates an inode for the vcpu.
1331  */
1332 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1333 {
1334         int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0);
1335         if (fd < 0)
1336                 kvm_put_kvm(vcpu->kvm);
1337         return fd;
1338 }
1339
1340 /*
1341  * Creates some virtual cpus.  Good luck creating more than one.
1342  */
1343 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
1344 {
1345         int r;
1346         struct kvm_vcpu *vcpu;
1347
1348         if (!valid_vcpu(n))
1349                 return -EINVAL;
1350
1351         vcpu = kvm_arch_vcpu_create(kvm, n);
1352         if (IS_ERR(vcpu))
1353                 return PTR_ERR(vcpu);
1354
1355         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1356
1357         r = kvm_arch_vcpu_setup(vcpu);
1358         if (r)
1359                 return r;
1360
1361         mutex_lock(&kvm->lock);
1362         if (kvm->vcpus[n]) {
1363                 r = -EEXIST;
1364                 goto vcpu_destroy;
1365         }
1366         kvm->vcpus[n] = vcpu;
1367         mutex_unlock(&kvm->lock);
1368
1369         /* Now it's all set up, let userspace reach it */
1370         kvm_get_kvm(kvm);
1371         r = create_vcpu_fd(vcpu);
1372         if (r < 0)
1373                 goto unlink;
1374         return r;
1375
1376 unlink:
1377         mutex_lock(&kvm->lock);
1378         kvm->vcpus[n] = NULL;
1379 vcpu_destroy:
1380         mutex_unlock(&kvm->lock);
1381         kvm_arch_vcpu_destroy(vcpu);
1382         return r;
1383 }
1384
1385 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1386 {
1387         if (sigset) {
1388                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1389                 vcpu->sigset_active = 1;
1390                 vcpu->sigset = *sigset;
1391         } else
1392                 vcpu->sigset_active = 0;
1393         return 0;
1394 }
1395
1396 static long kvm_vcpu_ioctl(struct file *filp,
1397                            unsigned int ioctl, unsigned long arg)
1398 {
1399         struct kvm_vcpu *vcpu = filp->private_data;
1400         void __user *argp = (void __user *)arg;
1401         int r;
1402         struct kvm_fpu *fpu = NULL;
1403         struct kvm_sregs *kvm_sregs = NULL;
1404
1405         if (vcpu->kvm->mm != current->mm)
1406                 return -EIO;
1407         switch (ioctl) {
1408         case KVM_RUN:
1409                 r = -EINVAL;
1410                 if (arg)
1411                         goto out;
1412                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1413                 break;
1414         case KVM_GET_REGS: {
1415                 struct kvm_regs *kvm_regs;
1416
1417                 r = -ENOMEM;
1418                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1419                 if (!kvm_regs)
1420                         goto out;
1421                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
1422                 if (r)
1423                         goto out_free1;
1424                 r = -EFAULT;
1425                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
1426                         goto out_free1;
1427                 r = 0;
1428 out_free1:
1429                 kfree(kvm_regs);
1430                 break;
1431         }
1432         case KVM_SET_REGS: {
1433                 struct kvm_regs *kvm_regs;
1434
1435                 r = -ENOMEM;
1436                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1437                 if (!kvm_regs)
1438                         goto out;
1439                 r = -EFAULT;
1440                 if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
1441                         goto out_free2;
1442                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
1443                 if (r)
1444                         goto out_free2;
1445                 r = 0;
1446 out_free2:
1447                 kfree(kvm_regs);
1448                 break;
1449         }
1450         case KVM_GET_SREGS: {
1451                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1452                 r = -ENOMEM;
1453                 if (!kvm_sregs)
1454                         goto out;
1455                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
1456                 if (r)
1457                         goto out;
1458                 r = -EFAULT;
1459                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
1460                         goto out;
1461                 r = 0;
1462                 break;
1463         }
1464         case KVM_SET_SREGS: {
1465                 kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1466                 r = -ENOMEM;
1467                 if (!kvm_sregs)
1468                         goto out;
1469                 r = -EFAULT;
1470                 if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
1471                         goto out;
1472                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
1473                 if (r)
1474                         goto out;
1475                 r = 0;
1476                 break;
1477         }
1478         case KVM_GET_MP_STATE: {
1479                 struct kvm_mp_state mp_state;
1480
1481                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
1482                 if (r)
1483                         goto out;
1484                 r = -EFAULT;
1485                 if (copy_to_user(argp, &mp_state, sizeof mp_state))
1486                         goto out;
1487                 r = 0;
1488                 break;
1489         }
1490         case KVM_SET_MP_STATE: {
1491                 struct kvm_mp_state mp_state;
1492
1493                 r = -EFAULT;
1494                 if (copy_from_user(&mp_state, argp, sizeof mp_state))
1495                         goto out;
1496                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
1497                 if (r)
1498                         goto out;
1499                 r = 0;
1500                 break;
1501         }
1502         case KVM_TRANSLATE: {
1503                 struct kvm_translation tr;
1504
1505                 r = -EFAULT;
1506                 if (copy_from_user(&tr, argp, sizeof tr))
1507                         goto out;
1508                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
1509                 if (r)
1510                         goto out;
1511                 r = -EFAULT;
1512                 if (copy_to_user(argp, &tr, sizeof tr))
1513                         goto out;
1514                 r = 0;
1515                 break;
1516         }
1517         case KVM_DEBUG_GUEST: {
1518                 struct kvm_debug_guest dbg;
1519
1520                 r = -EFAULT;
1521                 if (copy_from_user(&dbg, argp, sizeof dbg))
1522                         goto out;
1523                 r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
1524                 if (r)
1525                         goto out;
1526                 r = 0;
1527                 break;
1528         }
1529         case KVM_SET_SIGNAL_MASK: {
1530                 struct kvm_signal_mask __user *sigmask_arg = argp;
1531                 struct kvm_signal_mask kvm_sigmask;
1532                 sigset_t sigset, *p;
1533
1534                 p = NULL;
1535                 if (argp) {
1536                         r = -EFAULT;
1537                         if (copy_from_user(&kvm_sigmask, argp,
1538                                            sizeof kvm_sigmask))
1539                                 goto out;
1540                         r = -EINVAL;
1541                         if (kvm_sigmask.len != sizeof sigset)
1542                                 goto out;
1543                         r = -EFAULT;
1544                         if (copy_from_user(&sigset, sigmask_arg->sigset,
1545                                            sizeof sigset))
1546                                 goto out;
1547                         p = &sigset;
1548                 }
1549                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1550                 break;
1551         }
1552         case KVM_GET_FPU: {
1553                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1554                 r = -ENOMEM;
1555                 if (!fpu)
1556                         goto out;
1557                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
1558                 if (r)
1559                         goto out;
1560                 r = -EFAULT;
1561                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
1562                         goto out;
1563                 r = 0;
1564                 break;
1565         }
1566         case KVM_SET_FPU: {
1567                 fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1568                 r = -ENOMEM;
1569                 if (!fpu)
1570                         goto out;
1571                 r = -EFAULT;
1572                 if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
1573                         goto out;
1574                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
1575                 if (r)
1576                         goto out;
1577                 r = 0;
1578                 break;
1579         }
1580         default:
1581                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1582         }
1583 out:
1584         kfree(fpu);
1585         kfree(kvm_sregs);
1586         return r;
1587 }
1588
1589 static long kvm_vm_ioctl(struct file *filp,
1590                            unsigned int ioctl, unsigned long arg)
1591 {
1592         struct kvm *kvm = filp->private_data;
1593         void __user *argp = (void __user *)arg;
1594         int r;
1595
1596         if (kvm->mm != current->mm)
1597                 return -EIO;
1598         switch (ioctl) {
1599         case KVM_CREATE_VCPU:
1600                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1601                 if (r < 0)
1602                         goto out;
1603                 break;
1604         case KVM_SET_USER_MEMORY_REGION: {
1605                 struct kvm_userspace_memory_region kvm_userspace_mem;
1606
1607                 r = -EFAULT;
1608                 if (copy_from_user(&kvm_userspace_mem, argp,
1609                                                 sizeof kvm_userspace_mem))
1610                         goto out;
1611
1612                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
1613                 if (r)
1614                         goto out;
1615                 break;
1616         }
1617         case KVM_GET_DIRTY_LOG: {
1618                 struct kvm_dirty_log log;
1619
1620                 r = -EFAULT;
1621                 if (copy_from_user(&log, argp, sizeof log))
1622                         goto out;
1623                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
1624                 if (r)
1625                         goto out;
1626                 break;
1627         }
1628 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1629         case KVM_REGISTER_COALESCED_MMIO: {
1630                 struct kvm_coalesced_mmio_zone zone;
1631                 r = -EFAULT;
1632                 if (copy_from_user(&zone, argp, sizeof zone))
1633                         goto out;
1634                 r = -ENXIO;
1635                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
1636                 if (r)
1637                         goto out;
1638                 r = 0;
1639                 break;
1640         }
1641         case KVM_UNREGISTER_COALESCED_MMIO: {
1642                 struct kvm_coalesced_mmio_zone zone;
1643                 r = -EFAULT;
1644                 if (copy_from_user(&zone, argp, sizeof zone))
1645                         goto out;
1646                 r = -ENXIO;
1647                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
1648                 if (r)
1649                         goto out;
1650                 r = 0;
1651                 break;
1652         }
1653 #endif
1654 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
1655         case KVM_ASSIGN_PCI_DEVICE: {
1656                 struct kvm_assigned_pci_dev assigned_dev;
1657
1658                 r = -EFAULT;
1659                 if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
1660                         goto out;
1661                 r = kvm_vm_ioctl_assign_device(kvm, &assigned_dev);
1662                 if (r)
1663                         goto out;
1664                 break;
1665         }
1666         case KVM_ASSIGN_IRQ: {
1667                 struct kvm_assigned_irq assigned_irq;
1668
1669                 r = -EFAULT;
1670                 if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
1671                         goto out;
1672                 r = kvm_vm_ioctl_assign_irq(kvm, &assigned_irq);
1673                 if (r)
1674                         goto out;
1675                 break;
1676         }
1677 #endif
1678         default:
1679                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1680         }
1681 out:
1682         return r;
1683 }
1684
1685 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1686 {
1687         struct page *page[1];
1688         unsigned long addr;
1689         int npages;
1690         gfn_t gfn = vmf->pgoff;
1691         struct kvm *kvm = vma->vm_file->private_data;
1692
1693         addr = gfn_to_hva(kvm, gfn);
1694         if (kvm_is_error_hva(addr))
1695                 return VM_FAULT_SIGBUS;
1696
1697         npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
1698                                 NULL);
1699         if (unlikely(npages != 1))
1700                 return VM_FAULT_SIGBUS;
1701
1702         vmf->page = page[0];
1703         return 0;
1704 }
1705
1706 static struct vm_operations_struct kvm_vm_vm_ops = {
1707         .fault = kvm_vm_fault,
1708 };
1709
1710 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
1711 {
1712         vma->vm_ops = &kvm_vm_vm_ops;
1713         return 0;
1714 }
1715
1716 static const struct file_operations kvm_vm_fops = {
1717         .release        = kvm_vm_release,
1718         .unlocked_ioctl = kvm_vm_ioctl,
1719         .compat_ioctl   = kvm_vm_ioctl,
1720         .mmap           = kvm_vm_mmap,
1721 };
1722
1723 static int kvm_dev_ioctl_create_vm(void)
1724 {
1725         int fd;
1726         struct kvm *kvm;
1727
1728         kvm = kvm_create_vm();
1729         if (IS_ERR(kvm))
1730                 return PTR_ERR(kvm);
1731         fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, 0);
1732         if (fd < 0)
1733                 kvm_put_kvm(kvm);
1734
1735         return fd;
1736 }
1737
1738 static long kvm_dev_ioctl(struct file *filp,
1739                           unsigned int ioctl, unsigned long arg)
1740 {
1741         long r = -EINVAL;
1742
1743         switch (ioctl) {
1744         case KVM_GET_API_VERSION:
1745                 r = -EINVAL;
1746                 if (arg)
1747                         goto out;
1748                 r = KVM_API_VERSION;
1749                 break;
1750         case KVM_CREATE_VM:
1751                 r = -EINVAL;
1752                 if (arg)
1753                         goto out;
1754                 r = kvm_dev_ioctl_create_vm();
1755                 break;
1756         case KVM_CHECK_EXTENSION:
1757                 r = kvm_dev_ioctl_check_extension(arg);
1758                 break;
1759         case KVM_GET_VCPU_MMAP_SIZE:
1760                 r = -EINVAL;
1761                 if (arg)
1762                         goto out;
1763                 r = PAGE_SIZE;     /* struct kvm_run */
1764 #ifdef CONFIG_X86
1765                 r += PAGE_SIZE;    /* pio data page */
1766 #endif
1767 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1768                 r += PAGE_SIZE;    /* coalesced mmio ring page */
1769 #endif
1770                 break;
1771         case KVM_TRACE_ENABLE:
1772         case KVM_TRACE_PAUSE:
1773         case KVM_TRACE_DISABLE:
1774                 r = kvm_trace_ioctl(ioctl, arg);
1775                 break;
1776         default:
1777                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
1778         }
1779 out:
1780         return r;
1781 }
1782
1783 static struct file_operations kvm_chardev_ops = {
1784         .unlocked_ioctl = kvm_dev_ioctl,
1785         .compat_ioctl   = kvm_dev_ioctl,
1786 };
1787
1788 static struct miscdevice kvm_dev = {
1789         KVM_MINOR,
1790         "kvm",
1791         &kvm_chardev_ops,
1792 };
1793
1794 static void hardware_enable(void *junk)
1795 {
1796         int cpu = raw_smp_processor_id();
1797
1798         if (cpu_isset(cpu, cpus_hardware_enabled))
1799                 return;
1800         cpu_set(cpu, cpus_hardware_enabled);
1801         kvm_arch_hardware_enable(NULL);
1802 }
1803
1804 static void hardware_disable(void *junk)
1805 {
1806         int cpu = raw_smp_processor_id();
1807
1808         if (!cpu_isset(cpu, cpus_hardware_enabled))
1809                 return;
1810         cpu_clear(cpu, cpus_hardware_enabled);
1811         kvm_arch_hardware_disable(NULL);
1812 }
1813
1814 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
1815                            void *v)
1816 {
1817         int cpu = (long)v;
1818
1819         val &= ~CPU_TASKS_FROZEN;
1820         switch (val) {
1821         case CPU_DYING:
1822                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1823                        cpu);
1824                 hardware_disable(NULL);
1825                 break;
1826         case CPU_UP_CANCELED:
1827                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1828                        cpu);
1829                 smp_call_function_single(cpu, hardware_disable, NULL, 1);
1830                 break;
1831         case CPU_ONLINE:
1832                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
1833                        cpu);
1834                 smp_call_function_single(cpu, hardware_enable, NULL, 1);
1835                 break;
1836         }
1837         return NOTIFY_OK;
1838 }
1839
1840
1841 asmlinkage void kvm_handle_fault_on_reboot(void)
1842 {
1843         if (kvm_rebooting)
1844                 /* spin while reset goes on */
1845                 while (true)
1846                         ;
1847         /* Fault while not rebooting.  We want the trace. */
1848         BUG();
1849 }
1850 EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);
1851
1852 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
1853                       void *v)
1854 {
1855         if (val == SYS_RESTART) {
1856                 /*
1857                  * Some (well, at least mine) BIOSes hang on reboot if
1858                  * in vmx root mode.
1859                  */
1860                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1861                 kvm_rebooting = true;
1862                 on_each_cpu(hardware_disable, NULL, 1);
1863         }
1864         return NOTIFY_OK;
1865 }
1866
1867 static struct notifier_block kvm_reboot_notifier = {
1868         .notifier_call = kvm_reboot,
1869         .priority = 0,
1870 };
1871
1872 void kvm_io_bus_init(struct kvm_io_bus *bus)
1873 {
1874         memset(bus, 0, sizeof(*bus));
1875 }
1876
1877 void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1878 {
1879         int i;
1880
1881         for (i = 0; i < bus->dev_count; i++) {
1882                 struct kvm_io_device *pos = bus->devs[i];
1883
1884                 kvm_iodevice_destructor(pos);
1885         }
1886 }
1887
1888 struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus,
1889                                           gpa_t addr, int len, int is_write)
1890 {
1891         int i;
1892
1893         for (i = 0; i < bus->dev_count; i++) {
1894                 struct kvm_io_device *pos = bus->devs[i];
1895
1896                 if (pos->in_range(pos, addr, len, is_write))
1897                         return pos;
1898         }
1899
1900         return NULL;
1901 }
1902
1903 void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
1904 {
1905         BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
1906
1907         bus->devs[bus->dev_count++] = dev;
1908 }
1909
1910 static struct notifier_block kvm_cpu_notifier = {
1911         .notifier_call = kvm_cpu_hotplug,
1912         .priority = 20, /* must be > scheduler priority */
1913 };
1914
1915 static int vm_stat_get(void *_offset, u64 *val)
1916 {
1917         unsigned offset = (long)_offset;
1918         struct kvm *kvm;
1919
1920         *val = 0;
1921         spin_lock(&kvm_lock);
1922         list_for_each_entry(kvm, &vm_list, vm_list)
1923                 *val += *(u32 *)((void *)kvm + offset);
1924         spin_unlock(&kvm_lock);
1925         return 0;
1926 }
1927
1928 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
1929
1930 static int vcpu_stat_get(void *_offset, u64 *val)
1931 {
1932         unsigned offset = (long)_offset;
1933         struct kvm *kvm;
1934         struct kvm_vcpu *vcpu;
1935         int i;
1936
1937         *val = 0;
1938         spin_lock(&kvm_lock);
1939         list_for_each_entry(kvm, &vm_list, vm_list)
1940                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
1941                         vcpu = kvm->vcpus[i];
1942                         if (vcpu)
1943                                 *val += *(u32 *)((void *)vcpu + offset);
1944                 }
1945         spin_unlock(&kvm_lock);
1946         return 0;
1947 }
1948
1949 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
1950
1951 static struct file_operations *stat_fops[] = {
1952         [KVM_STAT_VCPU] = &vcpu_stat_fops,
1953         [KVM_STAT_VM]   = &vm_stat_fops,
1954 };
1955
1956 static void kvm_init_debug(void)
1957 {
1958         struct kvm_stats_debugfs_item *p;
1959
1960         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
1961         for (p = debugfs_entries; p->name; ++p)
1962                 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
1963                                                 (void *)(long)p->offset,
1964                                                 stat_fops[p->kind]);
1965 }
1966
1967 static void kvm_exit_debug(void)
1968 {
1969         struct kvm_stats_debugfs_item *p;
1970
1971         for (p = debugfs_entries; p->name; ++p)
1972                 debugfs_remove(p->dentry);
1973         debugfs_remove(kvm_debugfs_dir);
1974 }
1975
1976 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
1977 {
1978         hardware_disable(NULL);
1979         return 0;
1980 }
1981
1982 static int kvm_resume(struct sys_device *dev)
1983 {
1984         hardware_enable(NULL);
1985         return 0;
1986 }
1987
1988 static struct sysdev_class kvm_sysdev_class = {
1989         .name = "kvm",
1990         .suspend = kvm_suspend,
1991         .resume = kvm_resume,
1992 };
1993
1994 static struct sys_device kvm_sysdev = {
1995         .id = 0,
1996         .cls = &kvm_sysdev_class,
1997 };
1998
1999 struct page *bad_page;
2000 pfn_t bad_pfn;
2001
2002 static inline
2003 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2004 {
2005         return container_of(pn, struct kvm_vcpu, preempt_notifier);
2006 }
2007
2008 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2009 {
2010         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2011
2012         kvm_arch_vcpu_load(vcpu, cpu);
2013 }
2014
2015 static void kvm_sched_out(struct preempt_notifier *pn,
2016                           struct task_struct *next)
2017 {
2018         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2019
2020         kvm_arch_vcpu_put(vcpu);
2021 }
2022
2023 int kvm_init(void *opaque, unsigned int vcpu_size,
2024                   struct module *module)
2025 {
2026         int r;
2027         int cpu;
2028
2029         kvm_init_debug();
2030
2031         r = kvm_arch_init(opaque);
2032         if (r)
2033                 goto out_fail;
2034
2035         bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2036
2037         if (bad_page == NULL) {
2038                 r = -ENOMEM;
2039                 goto out;
2040         }
2041
2042         bad_pfn = page_to_pfn(bad_page);
2043
2044         r = kvm_arch_hardware_setup();
2045         if (r < 0)
2046                 goto out_free_0;
2047
2048         for_each_online_cpu(cpu) {
2049                 smp_call_function_single(cpu,
2050                                 kvm_arch_check_processor_compat,
2051                                 &r, 1);
2052                 if (r < 0)
2053                         goto out_free_1;
2054         }
2055
2056         on_each_cpu(hardware_enable, NULL, 1);
2057         r = register_cpu_notifier(&kvm_cpu_notifier);
2058         if (r)
2059                 goto out_free_2;
2060         register_reboot_notifier(&kvm_reboot_notifier);
2061
2062         r = sysdev_class_register(&kvm_sysdev_class);
2063         if (r)
2064                 goto out_free_3;
2065
2066         r = sysdev_register(&kvm_sysdev);
2067         if (r)
2068                 goto out_free_4;
2069
2070         /* A kmem cache lets us meet the alignment requirements of fx_save. */
2071         kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
2072                                            __alignof__(struct kvm_vcpu),
2073                                            0, NULL);
2074         if (!kvm_vcpu_cache) {
2075                 r = -ENOMEM;
2076                 goto out_free_5;
2077         }
2078
2079         kvm_chardev_ops.owner = module;
2080
2081         r = misc_register(&kvm_dev);
2082         if (r) {
2083                 printk(KERN_ERR "kvm: misc device register failed\n");
2084                 goto out_free;
2085         }
2086
2087         kvm_preempt_ops.sched_in = kvm_sched_in;
2088         kvm_preempt_ops.sched_out = kvm_sched_out;
2089
2090         return 0;
2091
2092 out_free:
2093         kmem_cache_destroy(kvm_vcpu_cache);
2094 out_free_5:
2095         sysdev_unregister(&kvm_sysdev);
2096 out_free_4:
2097         sysdev_class_unregister(&kvm_sysdev_class);
2098 out_free_3:
2099         unregister_reboot_notifier(&kvm_reboot_notifier);
2100         unregister_cpu_notifier(&kvm_cpu_notifier);
2101 out_free_2:
2102         on_each_cpu(hardware_disable, NULL, 1);
2103 out_free_1:
2104         kvm_arch_hardware_unsetup();
2105 out_free_0:
2106         __free_page(bad_page);
2107 out:
2108         kvm_arch_exit();
2109         kvm_exit_debug();
2110 out_fail:
2111         return r;
2112 }
2113 EXPORT_SYMBOL_GPL(kvm_init);
2114
2115 void kvm_exit(void)
2116 {
2117         kvm_trace_cleanup();
2118         misc_deregister(&kvm_dev);
2119         kmem_cache_destroy(kvm_vcpu_cache);
2120         sysdev_unregister(&kvm_sysdev);
2121         sysdev_class_unregister(&kvm_sysdev_class);
2122         unregister_reboot_notifier(&kvm_reboot_notifier);
2123         unregister_cpu_notifier(&kvm_cpu_notifier);
2124         on_each_cpu(hardware_disable, NULL, 1);
2125         kvm_arch_hardware_unsetup();
2126         kvm_arch_exit();
2127         kvm_exit_debug();
2128         __free_page(bad_page);
2129 }
2130 EXPORT_SYMBOL_GPL(kvm_exit);