KVM: MMU: invalidate and flush on spte small->large page size change

[safe/jmp/linux-2.6] / kernel / smp.c

diff --git a/kernel/smp.c b/kernel/smp.c
index a8c7606..75c970c 100644 (file)
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -9,18 +9,17 @@
 #include <linux/module.h>
 #include <linux/percpu.h>
 #include <linux/init.h>
+#include <linux/gfp.h>
 #include <linux/smp.h>
 #include <linux/cpu.h>
 
-static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
-
 static struct {
        struct list_head        queue;
-       spinlock_t              lock;
+       raw_spinlock_t          lock;
 } call_function __cacheline_aligned_in_smp =
        {
                .queue          = LIST_HEAD_INIT(call_function.queue),
-               .lock           = __SPIN_LOCK_UNLOCKED(call_function.lock),
+               .lock           = __RAW_SPIN_LOCK_UNLOCKED(call_function.lock),
        };
 
 enum {
@@ -33,12 +32,14 @@ struct call_function_data {
        cpumask_var_t           cpumask;
 };
 
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
+
 struct call_single_queue {
        struct list_head        list;
-       spinlock_t              lock;
+       raw_spinlock_t          lock;
 };
 
-static DEFINE_PER_CPU(struct call_function_data, cfd_data);
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
 
 static int
 hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
@@ -51,7 +52,7 @@ hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
        case CPU_UP_PREPARE_FROZEN:
                if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
                                cpu_to_node(cpu)))
-                       return NOTIFY_BAD;
+                       return notifier_from_errno(-ENOMEM);
                break;
 
 #ifdef CONFIG_HOTPLUG_CPU
@@ -80,7 +81,7 @@ static int __cpuinit init_call_single_data(void)
        for_each_possible_cpu(i) {
                struct call_single_queue *q = &per_cpu(call_single_queue, i);
 
-               spin_lock_init(&q->lock);
+               raw_spin_lock_init(&q->lock);
                INIT_LIST_HEAD(&q->list);
        }
 
@@ -141,10 +142,10 @@ void generic_exec_single(int cpu, struct call_single_data *data, int wait)
        unsigned long flags;
        int ipi;
 
-       spin_lock_irqsave(&dst->lock, flags);
+       raw_spin_lock_irqsave(&dst->lock, flags);
        ipi = list_empty(&dst->list);
        list_add_tail(&data->list, &dst->list);
-       spin_unlock_irqrestore(&dst->lock, flags);
+       raw_spin_unlock_irqrestore(&dst->lock, flags);
 
        /*
         * The list addition should be visible before sending the IPI
@@ -171,7 +172,7 @@ void generic_exec_single(int cpu, struct call_single_data *data, int wait)
 void generic_smp_call_function_interrupt(void)
 {
        struct call_function_data *data;
-       int cpu = get_cpu();
+       int cpu = smp_processor_id();
 
        /*
         * Shouldn't receive this interrupt on a cpu that is not yet online.
@@ -201,9 +202,9 @@ void generic_smp_call_function_interrupt(void)
                refs = atomic_dec_return(&data->refs);
                WARN_ON(refs < 0);
                if (!refs) {
-                       spin_lock(&call_function.lock);
+                       raw_spin_lock(&call_function.lock);
                        list_del_rcu(&data->csd.list);
-                       spin_unlock(&call_function.lock);
+                       raw_spin_unlock(&call_function.lock);
                }
 
                if (refs)
@@ -212,7 +213,6 @@ void generic_smp_call_function_interrupt(void)
                csd_unlock(&data->csd);
        }
 
-       put_cpu();
 }
 
 /*
@@ -230,9 +230,9 @@ void generic_smp_call_function_single_interrupt(void)
         */
        WARN_ON_ONCE(!cpu_online(smp_processor_id()));
 
-       spin_lock(&q->lock);
+       raw_spin_lock(&q->lock);
        list_replace_init(&q->list, &list);
-       spin_unlock(&q->lock);
+       raw_spin_unlock(&q->lock);
 
        while (!list_empty(&list)) {
                struct call_single_data *data;
@@ -257,7 +257,7 @@ void generic_smp_call_function_single_interrupt(void)
        }
 }
 
-static DEFINE_PER_CPU(struct call_single_data, csd_data);
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
 
 /*
  * smp_call_function_single - Run a function on a specific CPU
@@ -348,7 +348,7 @@ int smp_call_function_any(const struct cpumask *mask,
                goto call;
 
        /* Try for same node. */
-       nodemask = cpumask_of_node(cpu);
+       nodemask = cpumask_of_node(cpu_to_node(cpu));
        for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
             cpu = cpumask_next_and(cpu, nodemask, mask)) {
                if (cpu_online(cpu))
@@ -449,14 +449,14 @@ void smp_call_function_many(const struct cpumask *mask,
        cpumask_clear_cpu(this_cpu, data->cpumask);
        atomic_set(&data->refs, cpumask_weight(data->cpumask));
 
-       spin_lock_irqsave(&call_function.lock, flags);
+       raw_spin_lock_irqsave(&call_function.lock, flags);
        /*
         * Place entry at the _HEAD_ of the list, so that any cpu still
         * observing the entry in generic_smp_call_function_interrupt()
         * will not miss any other list entries:
         */
        list_add_rcu(&data->csd.list, &call_function.queue);
-       spin_unlock_irqrestore(&call_function.lock, flags);
+       raw_spin_unlock_irqrestore(&call_function.lock, flags);
 
        /*
         * Make the list addition visible before sending the ipi.
@@ -501,20 +501,20 @@ EXPORT_SYMBOL(smp_call_function);
 
 void ipi_call_lock(void)
 {
-       spin_lock(&call_function.lock);
+       raw_spin_lock(&call_function.lock);
 }
 
 void ipi_call_unlock(void)
 {
-       spin_unlock(&call_function.lock);
+       raw_spin_unlock(&call_function.lock);
 }
 
 void ipi_call_lock_irq(void)
 {
-       spin_lock_irq(&call_function.lock);
+       raw_spin_lock_irq(&call_function.lock);
 }
 
 void ipi_call_unlock_irq(void)
 {
-       spin_unlock_irq(&call_function.lock);
+       raw_spin_unlock_irq(&call_function.lock);
 }