on_each_cpu(): kill unused 'retry' parameter

[safe/jmp/linux-2.6] / kernel / softirq.c

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 0b9886a..c159fd0 100644 (file)
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -3,7 +3,9 @@
  *
  *     Copyright (C) 1992 Linus Torvalds
  *
- * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
+ *     Distribute under GPLv2.
+ *
+ *     Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
  */
 
 #include <linux/module.h>
@@ -14,6 +16,7 @@
 #include <linux/notifier.h>
 #include <linux/percpu.h>
 #include <linux/cpu.h>
+#include <linux/freezer.h>
 #include <linux/kthread.h>
 #include <linux/rcupdate.h>
 #include <linux/smp.h>
@@ -270,8 +273,6 @@ asmlinkage void do_softirq(void)
        local_irq_restore(flags);
 }
 
-EXPORT_SYMBOL(do_softirq);
-
 #endif
 
 /*
@@ -279,9 +280,14 @@ EXPORT_SYMBOL(do_softirq);
  */
 void irq_enter(void)
 {
+#ifdef CONFIG_NO_HZ
+       int cpu = smp_processor_id();
+       if (idle_cpu(cpu) && !in_interrupt())
+               tick_nohz_stop_idle(cpu);
+#endif
        __irq_enter();
 #ifdef CONFIG_NO_HZ
-       if (idle_cpu(smp_processor_id()))
+       if (idle_cpu(cpu))
                tick_nohz_update_jiffies();
 #endif
 }
@@ -307,6 +313,7 @@ void irq_exit(void)
        /* Make sure that timer wheel updates are propagated */
        if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
                tick_nohz_stop_sched_tick();
+       rcu_irq_exit();
 #endif
        preempt_enable_no_resched();
 }
@@ -314,7 +321,7 @@ void irq_exit(void)
 /*
  * This function must run with irqs disabled!
  */
-inline fastcall void raise_softirq_irqoff(unsigned int nr)
+inline void raise_softirq_irqoff(unsigned int nr)
 {
        __raise_softirq_irqoff(nr);
 
@@ -331,9 +338,7 @@ inline fastcall void raise_softirq_irqoff(unsigned int nr)
                wakeup_softirqd();
 }
 
-EXPORT_SYMBOL(raise_softirq_irqoff);
-
-void fastcall raise_softirq(unsigned int nr)
+void raise_softirq(unsigned int nr)
 {
        unsigned long flags;
 
@@ -351,7 +356,8 @@ void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 /* Tasklets */
 struct tasklet_head
 {
-       struct tasklet_struct *list;
+       struct tasklet_struct *head;
+       struct tasklet_struct **tail;
 };
 
 /* Some compilers disobey section attribute on statics when not
@@ -359,26 +365,28 @@ struct tasklet_head
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 
-void fastcall __tasklet_schedule(struct tasklet_struct *t)
+void __tasklet_schedule(struct tasklet_struct *t)
 {
        unsigned long flags;
 
        local_irq_save(flags);
-       t->next = __get_cpu_var(tasklet_vec).list;
-       __get_cpu_var(tasklet_vec).list = t;
+       t->next = NULL;
+       *__get_cpu_var(tasklet_vec).tail = t;
+       __get_cpu_var(tasklet_vec).tail = &(t->next);
        raise_softirq_irqoff(TASKLET_SOFTIRQ);
        local_irq_restore(flags);
 }
 
 EXPORT_SYMBOL(__tasklet_schedule);
 
-void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
+void __tasklet_hi_schedule(struct tasklet_struct *t)
 {
        unsigned long flags;
 
        local_irq_save(flags);
-       t->next = __get_cpu_var(tasklet_hi_vec).list;
-       __get_cpu_var(tasklet_hi_vec).list = t;
+       t->next = NULL;
+       *__get_cpu_var(tasklet_hi_vec).tail = t;
+       __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
        raise_softirq_irqoff(HI_SOFTIRQ);
        local_irq_restore(flags);
 }
@@ -390,8 +398,9 @@ static void tasklet_action(struct softirq_action *a)
        struct tasklet_struct *list;
 
        local_irq_disable();
-       list = __get_cpu_var(tasklet_vec).list;
-       __get_cpu_var(tasklet_vec).list = NULL;
+       list = __get_cpu_var(tasklet_vec).head;
+       __get_cpu_var(tasklet_vec).head = NULL;
+       __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
        local_irq_enable();
 
        while (list) {
@@ -411,8 +420,9 @@ static void tasklet_action(struct softirq_action *a)
                }
 
                local_irq_disable();
-               t->next = __get_cpu_var(tasklet_vec).list;
-               __get_cpu_var(tasklet_vec).list = t;
+               t->next = NULL;
+               *__get_cpu_var(tasklet_vec).tail = t;
+               __get_cpu_var(tasklet_vec).tail = &(t->next);
                __raise_softirq_irqoff(TASKLET_SOFTIRQ);
                local_irq_enable();
        }
@@ -423,8 +433,9 @@ static void tasklet_hi_action(struct softirq_action *a)
        struct tasklet_struct *list;
 
        local_irq_disable();
-       list = __get_cpu_var(tasklet_hi_vec).list;
-       __get_cpu_var(tasklet_hi_vec).list = NULL;
+       list = __get_cpu_var(tasklet_hi_vec).head;
+       __get_cpu_var(tasklet_hi_vec).head = NULL;
+       __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
        local_irq_enable();
 
        while (list) {
@@ -444,8 +455,9 @@ static void tasklet_hi_action(struct softirq_action *a)
                }
 
                local_irq_disable();
-               t->next = __get_cpu_var(tasklet_hi_vec).list;
-               __get_cpu_var(tasklet_hi_vec).list = t;
+               t->next = NULL;
+               *__get_cpu_var(tasklet_hi_vec).tail = t;
+               __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
                __raise_softirq_irqoff(HI_SOFTIRQ);
                local_irq_enable();
        }
@@ -482,15 +494,21 @@ EXPORT_SYMBOL(tasklet_kill);
 
 void __init softirq_init(void)
 {
+       int cpu;
+
+       for_each_possible_cpu(cpu) {
+               per_cpu(tasklet_vec, cpu).tail =
+                       &per_cpu(tasklet_vec, cpu).head;
+               per_cpu(tasklet_hi_vec, cpu).tail =
+                       &per_cpu(tasklet_hi_vec, cpu).head;
+       }
+
        open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
        open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 }
 
 static int ksoftirqd(void * __bind_cpu)
 {
-       set_user_nice(current, 19);
-       current->flags |= PF_NOFREEZE;
-
        set_current_state(TASK_INTERRUPTIBLE);
 
        while (!kthread_should_stop()) {
@@ -553,9 +571,12 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
                return;
 
        /* CPU is dead, so no lock needed. */
-       for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
+       for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
                if (*i == t) {
                        *i = t->next;
+                       /* If this was the tail element, move the tail ptr */
+                       if (*i == NULL)
+                               per_cpu(tasklet_vec, cpu).tail = i;
                        return;
                }
        }
@@ -564,20 +585,24 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 
 static void takeover_tasklets(unsigned int cpu)
 {
-       struct tasklet_struct **i;
-
        /* CPU is dead, so no lock needed. */
        local_irq_disable();
 
        /* Find end, append list for that CPU. */
-       for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
-       *i = per_cpu(tasklet_vec, cpu).list;
-       per_cpu(tasklet_vec, cpu).list = NULL;
+       if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
+               *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
+               __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
+               per_cpu(tasklet_vec, cpu).head = NULL;
+               per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
+       }
        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 
-       for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
-       *i = per_cpu(tasklet_hi_vec, cpu).list;
-       per_cpu(tasklet_hi_vec, cpu).list = NULL;
+       if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
+               *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
+               __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
+               per_cpu(tasklet_hi_vec, cpu).head = NULL;
+               per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
+       }
        raise_softirq_irqoff(HI_SOFTIRQ);
 
        local_irq_enable();
@@ -615,12 +640,16 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb,
                kthread_bind(per_cpu(ksoftirqd, hotcpu),
                             any_online_cpu(cpu_online_map));
        case CPU_DEAD:
-       case CPU_DEAD_FROZEN:
+       case CPU_DEAD_FROZEN: {
+               struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
+
                p = per_cpu(ksoftirqd, hotcpu);
                per_cpu(ksoftirqd, hotcpu) = NULL;
+               sched_setscheduler(p, SCHED_FIFO, &param);
                kthread_stop(p);
                takeover_tasklets(hotcpu);
                break;
+       }
 #endif /* CONFIG_HOTPLUG_CPU */
        }
        return NOTIFY_OK;
@@ -645,12 +674,12 @@ __init int spawn_ksoftirqd(void)
 /*
  * Call a function on all processors
  */
-int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
+int on_each_cpu(void (*func) (void *info), void *info, int wait)
 {
        int ret = 0;
 
        preempt_disable();
-       ret = smp_call_function(func, info, retry, wait);
+       ret = smp_call_function(func, info, wait);
        local_irq_disable();
        func(info);
        local_irq_enable();