sched: clear buddies more aggressively

[safe/jmp/linux-2.6] / kernel / rcupreempt.c

diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
index aaa7976..33cfc50 100644 (file)
--- a/kernel/rcupreempt.c
+++ b/kernel/rcupreempt.c
@@ -51,21 +51,12 @@
 #include <linux/moduleparam.h>
 #include <linux/percpu.h>
 #include <linux/notifier.h>
-#include <linux/rcupdate.h>
 #include <linux/cpu.h>
 #include <linux/random.h>
 #include <linux/delay.h>
-#include <linux/byteorder/swabb.h>
 #include <linux/cpumask.h>
 #include <linux/rcupreempt_trace.h>
-
-/*
- * Macro that prevents the compiler from reordering accesses, but does
- * absolutely -nothing- to prevent CPUs from reordering.  This is used
- * only to mediate communication between mainline code and hardware
- * interrupt and NMI handlers.
- */
-#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
+#include <asm/byteorder.h>
 
 /*
  * PREEMPT_RCU data structures.
@@ -83,7 +74,6 @@ struct rcu_data {
        spinlock_t      lock;           /* Protect rcu_data fields. */
        long            completed;      /* Number of last completed batch. */
        int             waitlistcount;
-       struct tasklet_struct rcu_tasklet;
        struct rcu_head *nextlist;
        struct rcu_head **nexttail;
        struct rcu_head *waitlist[GP_STAGES];
@@ -174,7 +164,8 @@ static char *rcu_try_flip_state_names[] =
        { "idle", "waitack", "waitzero", "waitmb" };
 #endif /* #ifdef CONFIG_RCU_TRACE */
 
-static cpumask_t rcu_cpu_online_map __read_mostly = CPU_MASK_NONE;
+static DECLARE_BITMAP(rcu_cpu_online_map, NR_CPUS) __read_mostly
+       = CPU_BITS_NONE;
 
 /*
  * Enum and per-CPU flag to determine when each CPU has seen
@@ -561,6 +552,16 @@ void rcu_irq_exit(void)
        }
 }
 
+void rcu_nmi_enter(void)
+{
+       rcu_irq_enter();
+}
+
+void rcu_nmi_exit(void)
+{
+       rcu_irq_exit();
+}
+
 static void dyntick_save_progress_counter(int cpu)
 {
        struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
@@ -597,7 +598,7 @@ rcu_try_flip_waitack_needed(int cpu)
         * that this CPU already acknowledged the counter.
         */
 
-       if ((curr - snap) > 2 || (snap & 0x1) == 0)
+       if ((curr - snap) > 2 || (curr & 0x1) == 0)
                return 0;
 
        /* We need this CPU to explicitly acknowledge the counter flip. */
@@ -758,7 +759,7 @@ rcu_try_flip_idle(void)
 
        /* Now ask each CPU for acknowledgement of the flip. */
 
-       for_each_cpu_mask(cpu, rcu_cpu_online_map) {
+       for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
                per_cpu(rcu_flip_flag, cpu) = rcu_flipped;
                dyntick_save_progress_counter(cpu);
        }
@@ -776,7 +777,7 @@ rcu_try_flip_waitack(void)
        int cpu;
 
        RCU_TRACE_ME(rcupreempt_trace_try_flip_a1);
-       for_each_cpu_mask(cpu, rcu_cpu_online_map)
+       for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
                if (rcu_try_flip_waitack_needed(cpu) &&
                    per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {
                        RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1);
@@ -808,7 +809,7 @@ rcu_try_flip_waitzero(void)
        /* Check to see if the sum of the "last" counters is zero. */
 
        RCU_TRACE_ME(rcupreempt_trace_try_flip_z1);
-       for_each_cpu_mask(cpu, rcu_cpu_online_map)
+       for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
                sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx];
        if (sum != 0) {
                RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1);
@@ -823,7 +824,7 @@ rcu_try_flip_waitzero(void)
        smp_mb();  /*  ^^^^^^^^^^^^ */
 
        /* Call for a memory barrier from each CPU. */
-       for_each_cpu_mask(cpu, rcu_cpu_online_map) {
+       for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
                per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;
                dyntick_save_progress_counter(cpu);
        }
@@ -843,7 +844,7 @@ rcu_try_flip_waitmb(void)
        int cpu;
 
        RCU_TRACE_ME(rcupreempt_trace_try_flip_m1);
-       for_each_cpu_mask(cpu, rcu_cpu_online_map)
+       for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
                if (rcu_try_flip_waitmb_needed(cpu) &&
                    per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {
                        RCU_TRACE_ME(rcupreempt_trace_try_flip_me1);
@@ -1032,7 +1033,7 @@ void rcu_offline_cpu(int cpu)
        RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0;
        RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0;
 
-       cpu_clear(cpu, rcu_cpu_online_map);
+       cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map));
 
        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
 
@@ -1058,13 +1059,21 @@ void rcu_offline_cpu(int cpu)
        spin_unlock_irqrestore(&rdp->lock, flags);
 }
 
-void __devinit rcu_online_cpu(int cpu)
+#else /* #ifdef CONFIG_HOTPLUG_CPU */
+
+void rcu_offline_cpu(int cpu)
+{
+}
+
+#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
+
+void __cpuinit rcu_online_cpu(int cpu)
 {
        unsigned long flags;
        struct rcu_data *rdp;
 
        spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
-       cpu_set(cpu, rcu_cpu_online_map);
+       cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map));
        spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
 
        /*
@@ -1082,18 +1091,6 @@ void __devinit rcu_online_cpu(int cpu)
        spin_unlock_irqrestore(&rdp->lock, flags);
 }
 
-#else /* #ifdef CONFIG_HOTPLUG_CPU */
-
-void rcu_offline_cpu(int cpu)
-{
-}
-
-void __devinit rcu_online_cpu(int cpu)
-{
-}
-
-#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
-
 static void rcu_process_callbacks(struct softirq_action *unused)
 {
        unsigned long flags;
@@ -1180,7 +1177,16 @@ EXPORT_SYMBOL_GPL(call_rcu_sched);
  * in -rt this does -not- necessarily result in all currently executing
  * interrupt -handlers- having completed.
  */
-synchronize_rcu_xxx(__synchronize_sched, call_rcu_sched)
+void __synchronize_sched(void)
+{
+       struct rcu_synchronize rcu;
+
+       init_completion(&rcu.completion);
+       /* Will wake me after RCU finished. */
+       call_rcu_sched(&rcu.head, wakeme_after_rcu);
+       /* Wait for it. */
+       wait_for_completion(&rcu.completion);
+}
 EXPORT_SYMBOL_GPL(__synchronize_sched);
 
 /*
@@ -1434,7 +1440,7 @@ void __init __rcu_init(void)
         * We don't need protection against CPU-Hotplug here
         * since
         * a) If a CPU comes online while we are iterating over the
-        *    cpu_online_map below, we would only end up making a
+        *    cpu_online_mask below, we would only end up making a
         *    duplicate call to rcu_online_cpu() which sets the corresponding
         *    CPU's mask in the rcu_cpu_online_map.
         *
@@ -1445,7 +1451,7 @@ void __init __rcu_init(void)
        for_each_online_cpu(cpu)
                rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu);
 
-       open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL);
+       open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 }
 
 /*