SAFE public projects git trees. - safe/jmp/linux-2.6/blob - kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/tick.h>
  27
  28 #include <asm/irq.h>
  29 /*
  30    - No shared variables, all the data are CPU local.
  31    - If a softirq needs serialization, let it serialize itself
  32      by its own spinlocks.
  33    - Even if softirq is serialized, only local cpu is marked for
  34      execution. Hence, we get something sort of weak cpu binding.
  35      Though it is still not clear, will it result in better locality
  36      or will not.
  37
  38    Examples:
  39    - NET RX softirq. It is multithreaded and does not require
  40      any global serialization.
  41    - NET TX softirq. It kicks software netdevice queues, hence
  42      it is logically serialized per device, but this serialization
  43      is invisible to common code.
  44    - Tasklets: serialized wrt itself.
  45  */
  46
  47 #ifndef __ARCH_IRQ_STAT
  48 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  49 EXPORT_SYMBOL(irq_stat);
  50 #endif
  51
  52 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  53
  54 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  55
  56 char *softirq_to_name[NR_SOFTIRQS] = {
  57         "HI_SOFTIRQ", "TIMER_SOFTIRQ", "NET_TX_SOFTIRQ", "NET_RX_SOFTIRQ",
  58         "BLOCK_SOFTIRQ", "TASKLET_SOFTIRQ", "SCHED_SOFTIRQ", "HRTIMER_SOFTIRQ",
  59         "RCU_SOFTIRQ"
  60 };
  61
  62 /*
  63  * we cannot loop indefinitely here to avoid userspace starvation,
  64  * but we also don't want to introduce a worst case 1/HZ latency
  65  * to the pending events, so lets the scheduler to balance
  66  * the softirq load for us.
  67  */
  68 static inline void wakeup_softirqd(void)
  69 {
  70         /* Interrupts are disabled: no need to stop preemption */
  71         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  72
  73         if (tsk && tsk->state != TASK_RUNNING)
  74                 wake_up_process(tsk);
  75 }
  76
  77 /*
  78  * This one is for softirq.c-internal use,
  79  * where hardirqs are disabled legitimately:
  80  */
  81 #ifdef CONFIG_TRACE_IRQFLAGS
  82 static void __local_bh_disable(unsigned long ip)
  83 {
  84         unsigned long flags;
  85
  86         WARN_ON_ONCE(in_irq());
  87
  88         raw_local_irq_save(flags);
  89         /*
  90          * The preempt tracer hooks into add_preempt_count and will break
  91          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
  92          * is set and before current->softirq_enabled is cleared.
  93          * We must manually increment preempt_count here and manually
  94          * call the trace_preempt_off later.
  95          */
  96         preempt_count() += SOFTIRQ_OFFSET;
  97         /*
  98          * Were softirqs turned off above:
  99          */
 100         if (softirq_count() == SOFTIRQ_OFFSET)
 101                 trace_softirqs_off(ip);
 102         raw_local_irq_restore(flags);
 103
 104         if (preempt_count() == SOFTIRQ_OFFSET)
 105                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 106 }
 107 #else /* !CONFIG_TRACE_IRQFLAGS */
 108 static inline void __local_bh_disable(unsigned long ip)
 109 {
 110         add_preempt_count(SOFTIRQ_OFFSET);
 111         barrier();
 112 }
 113 #endif /* CONFIG_TRACE_IRQFLAGS */
 114
 115 void local_bh_disable(void)
 116 {
 117         __local_bh_disable((unsigned long)__builtin_return_address(0));
 118 }
 119
 120 EXPORT_SYMBOL(local_bh_disable);
 121
 122 /*
 123  * Special-case - softirqs can safely be enabled in
 124  * cond_resched_softirq(), or by __do_softirq(),
 125  * without processing still-pending softirqs:
 126  */
 127 void _local_bh_enable(void)
 128 {
 129         WARN_ON_ONCE(in_irq());
 130         WARN_ON_ONCE(!irqs_disabled());
 131
 132         if (softirq_count() == SOFTIRQ_OFFSET)
 133                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 134         sub_preempt_count(SOFTIRQ_OFFSET);
 135 }
 136
 137 EXPORT_SYMBOL(_local_bh_enable);
 138
 139 static inline void _local_bh_enable_ip(unsigned long ip)
 140 {
 141         WARN_ON_ONCE(in_irq() || irqs_disabled());
 142 #ifdef CONFIG_TRACE_IRQFLAGS
 143         local_irq_disable();
 144 #endif
 145         /*
 146          * Are softirqs going to be turned on now:
 147          */
 148         if (softirq_count() == SOFTIRQ_OFFSET)
 149                 trace_softirqs_on(ip);
 150         /*
 151          * Keep preemption disabled until we are done with
 152          * softirq processing:
 153          */
 154         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 155
 156         if (unlikely(!in_interrupt() && local_softirq_pending()))
 157                 do_softirq();
 158
 159         dec_preempt_count();
 160 #ifdef CONFIG_TRACE_IRQFLAGS
 161         local_irq_enable();
 162 #endif
 163         preempt_check_resched();
 164 }
 165
 166 void local_bh_enable(void)
 167 {
 168         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 169 }
 170 EXPORT_SYMBOL(local_bh_enable);
 171
 172 void local_bh_enable_ip(unsigned long ip)
 173 {
 174         _local_bh_enable_ip(ip);
 175 }
 176 EXPORT_SYMBOL(local_bh_enable_ip);
 177
 178 /*
 179  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 180  * and we fall back to softirqd after that.
 181  *
 182  * This number has been established via experimentation.
 183  * The two things to balance is latency against fairness -
 184  * we want to handle softirqs as soon as possible, but they
 185  * should not be able to lock up the box.
 186  */
 187 #define MAX_SOFTIRQ_RESTART 10
 188
 189 asmlinkage void __do_softirq(void)
 190 {
 191         struct softirq_action *h;
 192         __u32 pending;
 193         int max_restart = MAX_SOFTIRQ_RESTART;
 194         int cpu;
 195
 196         pending = local_softirq_pending();
 197         account_system_vtime(current);
 198
 199         __local_bh_disable((unsigned long)__builtin_return_address(0));
 200         lockdep_softirq_enter();
 201
 202         cpu = smp_processor_id();
 203 restart:
 204         /* Reset the pending bitmask before enabling irqs */
 205         set_softirq_pending(0);
 206
 207         local_irq_enable();
 208
 209         h = softirq_vec;
 210
 211         do {
 212                 if (pending & 1) {
 213                         int prev_count = preempt_count();
 214
 215                         h->action(h);
 216
 217                         if (unlikely(prev_count != preempt_count())) {
 218                                 printk(KERN_ERR "huh, entered softirq %td %s %p"
 219                                        "with preempt_count %08x,"
 220                                        " exited with %08x?\n", h - softirq_vec,
 221                                        softirq_to_name[h - softirq_vec],
 222                                        h->action, prev_count, preempt_count());
 223                                 preempt_count() = prev_count;
 224                         }
 225
 226                         rcu_bh_qsctr_inc(cpu);
 227                 }
 228                 h++;
 229                 pending >>= 1;
 230         } while (pending);
 231
 232         local_irq_disable();
 233
 234         pending = local_softirq_pending();
 235         if (pending && --max_restart)
 236                 goto restart;
 237
 238         if (pending)
 239                 wakeup_softirqd();
 240
 241         lockdep_softirq_exit();
 242
 243         account_system_vtime(current);
 244         _local_bh_enable();
 245 }
 246
 247 #ifndef __ARCH_HAS_DO_SOFTIRQ
 248
 249 asmlinkage void do_softirq(void)
 250 {
 251         __u32 pending;
 252         unsigned long flags;
 253
 254         if (in_interrupt())
 255                 return;
 256
 257         local_irq_save(flags);
 258
 259         pending = local_softirq_pending();
 260
 261         if (pending)
 262                 __do_softirq();
 263
 264         local_irq_restore(flags);
 265 }
 266
 267 #endif
 268
 269 /*
 270  * Enter an interrupt context.
 271  */
 272 void irq_enter(void)
 273 {
 274         int cpu = smp_processor_id();
 275
 276         rcu_irq_enter();
 277         if (idle_cpu(cpu) && !in_interrupt()) {
 278                 __irq_enter();
 279                 tick_check_idle(cpu);
 280         } else
 281                 __irq_enter();
 282 }
 283
 284 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 285 # define invoke_softirq()       __do_softirq()
 286 #else
 287 # define invoke_softirq()       do_softirq()
 288 #endif
 289
 290 /*
 291  * Exit an interrupt context. Process softirqs if needed and possible:
 292  */
 293 void irq_exit(void)
 294 {
 295         account_system_vtime(current);
 296         trace_hardirq_exit();
 297         sub_preempt_count(IRQ_EXIT_OFFSET);
 298         if (!in_interrupt() && local_softirq_pending())
 299                 invoke_softirq();
 300
 301 #ifdef CONFIG_NO_HZ
 302         /* Make sure that timer wheel updates are propagated */
 303         rcu_irq_exit();
 304         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 305                 tick_nohz_stop_sched_tick(0);
 306 #endif
 307         preempt_enable_no_resched();
 308 }
 309
 310 /*
 311  * This function must run with irqs disabled!
 312  */
 313 inline void raise_softirq_irqoff(unsigned int nr)
 314 {
 315         __raise_softirq_irqoff(nr);
 316
 317         /*
 318          * If we're in an interrupt or softirq, we're done
 319          * (this also catches softirq-disabled code). We will
 320          * actually run the softirq once we return from
 321          * the irq or softirq.
 322          *
 323          * Otherwise we wake up ksoftirqd to make sure we
 324          * schedule the softirq soon.
 325          */
 326         if (!in_interrupt())
 327                 wakeup_softirqd();
 328 }
 329
 330 void raise_softirq(unsigned int nr)
 331 {
 332         unsigned long flags;
 333
 334         local_irq_save(flags);
 335         raise_softirq_irqoff(nr);
 336         local_irq_restore(flags);
 337 }
 338
 339 void open_softirq(int nr, void (*action)(struct softirq_action *))
 340 {
 341         softirq_vec[nr].action = action;
 342 }
 343
 344 /* Tasklets */
 345 struct tasklet_head
 346 {
 347         struct tasklet_struct *head;
 348         struct tasklet_struct **tail;
 349 };
 350
 351 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 352 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 353
 354 void __tasklet_schedule(struct tasklet_struct *t)
 355 {
 356         unsigned long flags;
 357
 358         local_irq_save(flags);
 359         t->next = NULL;
 360         *__get_cpu_var(tasklet_vec).tail = t;
 361         __get_cpu_var(tasklet_vec).tail = &(t->next);
 362         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 363         local_irq_restore(flags);
 364 }
 365
 366 EXPORT_SYMBOL(__tasklet_schedule);
 367
 368 void __tasklet_hi_schedule(struct tasklet_struct *t)
 369 {
 370         unsigned long flags;
 371
 372         local_irq_save(flags);
 373         t->next = NULL;
 374         *__get_cpu_var(tasklet_hi_vec).tail = t;
 375         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 376         raise_softirq_irqoff(HI_SOFTIRQ);
 377         local_irq_restore(flags);
 378 }
 379
 380 EXPORT_SYMBOL(__tasklet_hi_schedule);
 381
 382 static void tasklet_action(struct softirq_action *a)
 383 {
 384         struct tasklet_struct *list;
 385
 386         local_irq_disable();
 387         list = __get_cpu_var(tasklet_vec).head;
 388         __get_cpu_var(tasklet_vec).head = NULL;
 389         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 390         local_irq_enable();
 391
 392         while (list) {
 393                 struct tasklet_struct *t = list;
 394
 395                 list = list->next;
 396
 397                 if (tasklet_trylock(t)) {
 398                         if (!atomic_read(&t->count)) {
 399                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 400                                         BUG();
 401                                 t->func(t->data);
 402                                 tasklet_unlock(t);
 403                                 continue;
 404                         }
 405                         tasklet_unlock(t);
 406                 }
 407
 408                 local_irq_disable();
 409                 t->next = NULL;
 410                 *__get_cpu_var(tasklet_vec).tail = t;
 411                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 412                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 413                 local_irq_enable();
 414         }
 415 }
 416
 417 static void tasklet_hi_action(struct softirq_action *a)
 418 {
 419         struct tasklet_struct *list;
 420
 421         local_irq_disable();
 422         list = __get_cpu_var(tasklet_hi_vec).head;
 423         __get_cpu_var(tasklet_hi_vec).head = NULL;
 424         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 425         local_irq_enable();
 426
 427         while (list) {
 428                 struct tasklet_struct *t = list;
 429
 430                 list = list->next;
 431
 432                 if (tasklet_trylock(t)) {
 433                         if (!atomic_read(&t->count)) {
 434                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 435                                         BUG();
 436                                 t->func(t->data);
 437                                 tasklet_unlock(t);
 438                                 continue;
 439                         }
 440                         tasklet_unlock(t);
 441                 }
 442
 443                 local_irq_disable();
 444                 t->next = NULL;
 445                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 446                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 447                 __raise_softirq_irqoff(HI_SOFTIRQ);
 448                 local_irq_enable();
 449         }
 450 }
 451
 452
 453 void tasklet_init(struct tasklet_struct *t,
 454                   void (*func)(unsigned long), unsigned long data)
 455 {
 456         t->next = NULL;
 457         t->state = 0;
 458         atomic_set(&t->count, 0);
 459         t->func = func;
 460         t->data = data;
 461 }
 462
 463 EXPORT_SYMBOL(tasklet_init);
 464
 465 void tasklet_kill(struct tasklet_struct *t)
 466 {
 467         if (in_interrupt())
 468                 printk("Attempt to kill tasklet from interrupt\n");
 469
 470         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 471                 do
 472                         yield();
 473                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 474         }
 475         tasklet_unlock_wait(t);
 476         clear_bit(TASKLET_STATE_SCHED, &t->state);
 477 }
 478
 479 EXPORT_SYMBOL(tasklet_kill);
 480
 481 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 482 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 483
 484 static void __local_trigger(struct call_single_data *cp, int softirq)
 485 {
 486         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 487
 488         list_add_tail(&cp->list, head);
 489
 490         /* Trigger the softirq only if the list was previously empty.  */
 491         if (head->next == &cp->list)
 492                 raise_softirq_irqoff(softirq);
 493 }
 494
 495 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 496 static void remote_softirq_receive(void *data)
 497 {
 498         struct call_single_data *cp = data;
 499         unsigned long flags;
 500         int softirq;
 501
 502         softirq = cp->priv;
 503
 504         local_irq_save(flags);
 505         __local_trigger(cp, softirq);
 506         local_irq_restore(flags);
 507 }
 508
 509 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 510 {
 511         if (cpu_online(cpu)) {
 512                 cp->func = remote_softirq_receive;
 513                 cp->info = cp;
 514                 cp->flags = 0;
 515                 cp->priv = softirq;
 516
 517                 __smp_call_function_single(cpu, cp);
 518                 return 0;
 519         }
 520         return 1;
 521 }
 522 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 523 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 524 {
 525         return 1;
 526 }
 527 #endif
 528
 529 /**
 530  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 531  * @cp: private SMP call function data area
 532  * @cpu: the remote cpu
 533  * @this_cpu: the currently executing cpu
 534  * @softirq: the softirq for the work
 535  *
 536  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 537  * done, the work is instead queued up on the local cpu.
 538  *
 539  * Interrupts must be disabled.
 540  */
 541 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 542 {
 543         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 544                 __local_trigger(cp, softirq);
 545 }
 546 EXPORT_SYMBOL(__send_remote_softirq);
 547
 548 /**
 549  * send_remote_softirq - try to schedule softirq work on a remote cpu
 550  * @cp: private SMP call function data area
 551  * @cpu: the remote cpu
 552  * @softirq: the softirq for the work
 553  *
 554  * Like __send_remote_softirq except that disabling interrupts and
 555  * computing the current cpu is done for the caller.
 556  */
 557 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 558 {
 559         unsigned long flags;
 560         int this_cpu;
 561
 562         local_irq_save(flags);
 563         this_cpu = smp_processor_id();
 564         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 565         local_irq_restore(flags);
 566 }
 567 EXPORT_SYMBOL(send_remote_softirq);
 568
 569 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 570                                                unsigned long action, void *hcpu)
 571 {
 572         /*
 573          * If a CPU goes away, splice its entries to the current CPU
 574          * and trigger a run of the softirq
 575          */
 576         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 577                 int cpu = (unsigned long) hcpu;
 578                 int i;
 579
 580                 local_irq_disable();
 581                 for (i = 0; i < NR_SOFTIRQS; i++) {
 582                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 583                         struct list_head *local_head;
 584
 585                         if (list_empty(head))
 586                                 continue;
 587
 588                         local_head = &__get_cpu_var(softirq_work_list[i]);
 589                         list_splice_init(head, local_head);
 590                         raise_softirq_irqoff(i);
 591                 }
 592                 local_irq_enable();
 593         }
 594
 595         return NOTIFY_OK;
 596 }
 597
 598 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 599         .notifier_call  = remote_softirq_cpu_notify,
 600 };
 601
 602 void __init softirq_init(void)
 603 {
 604         int cpu;
 605
 606         for_each_possible_cpu(cpu) {
 607                 int i;
 608
 609                 per_cpu(tasklet_vec, cpu).tail =
 610                         &per_cpu(tasklet_vec, cpu).head;
 611                 per_cpu(tasklet_hi_vec, cpu).tail =
 612                         &per_cpu(tasklet_hi_vec, cpu).head;
 613                 for (i = 0; i < NR_SOFTIRQS; i++)
 614                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 615         }
 616
 617         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 618
 619         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 620         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 621 }
 622
 623 static int ksoftirqd(void * __bind_cpu)
 624 {
 625         set_current_state(TASK_INTERRUPTIBLE);
 626
 627         while (!kthread_should_stop()) {
 628                 preempt_disable();
 629                 if (!local_softirq_pending()) {
 630                         preempt_enable_no_resched();
 631                         schedule();
 632                         preempt_disable();
 633                 }
 634
 635                 __set_current_state(TASK_RUNNING);
 636
 637                 while (local_softirq_pending()) {
 638                         /* Preempt disable stops cpu going offline.
 639                            If already offline, we'll be on wrong CPU:
 640                            don't process */
 641                         if (cpu_is_offline((long)__bind_cpu))
 642                                 goto wait_to_die;
 643                         do_softirq();
 644                         preempt_enable_no_resched();
 645                         cond_resched();
 646                         preempt_disable();
 647                         rcu_qsctr_inc((long)__bind_cpu);
 648                 }
 649                 preempt_enable();
 650                 set_current_state(TASK_INTERRUPTIBLE);
 651         }
 652         __set_current_state(TASK_RUNNING);
 653         return 0;
 654
 655 wait_to_die:
 656         preempt_enable();
 657         /* Wait for kthread_stop */
 658         set_current_state(TASK_INTERRUPTIBLE);
 659         while (!kthread_should_stop()) {
 660                 schedule();
 661                 set_current_state(TASK_INTERRUPTIBLE);
 662         }
 663         __set_current_state(TASK_RUNNING);
 664         return 0;
 665 }
 666
 667 #ifdef CONFIG_HOTPLUG_CPU
 668 /*
 669  * tasklet_kill_immediate is called to remove a tasklet which can already be
 670  * scheduled for execution on @cpu.
 671  *
 672  * Unlike tasklet_kill, this function removes the tasklet
 673  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 674  *
 675  * When this function is called, @cpu must be in the CPU_DEAD state.
 676  */
 677 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 678 {
 679         struct tasklet_struct **i;
 680
 681         BUG_ON(cpu_online(cpu));
 682         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 683
 684         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 685                 return;
 686
 687         /* CPU is dead, so no lock needed. */
 688         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 689                 if (*i == t) {
 690                         *i = t->next;
 691                         /* If this was the tail element, move the tail ptr */
 692                         if (*i == NULL)
 693                                 per_cpu(tasklet_vec, cpu).tail = i;
 694                         return;
 695                 }
 696         }
 697         BUG();
 698 }
 699
 700 static void takeover_tasklets(unsigned int cpu)
 701 {
 702         /* CPU is dead, so no lock needed. */
 703         local_irq_disable();
 704
 705         /* Find end, append list for that CPU. */
 706         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 707                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 708                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 709                 per_cpu(tasklet_vec, cpu).head = NULL;
 710                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 711         }
 712         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 713
 714         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 715                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 716                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 717                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 718                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 719         }
 720         raise_softirq_irqoff(HI_SOFTIRQ);
 721
 722         local_irq_enable();
 723 }
 724 #endif /* CONFIG_HOTPLUG_CPU */
 725
 726 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 727                                   unsigned long action,
 728                                   void *hcpu)
 729 {
 730         int hotcpu = (unsigned long)hcpu;
 731         struct task_struct *p;
 732
 733         switch (action) {
 734         case CPU_UP_PREPARE:
 735         case CPU_UP_PREPARE_FROZEN:
 736                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 737                 if (IS_ERR(p)) {
 738                         printk("ksoftirqd for %i failed\n", hotcpu);
 739                         return NOTIFY_BAD;
 740                 }
 741                 kthread_bind(p, hotcpu);
 742                 per_cpu(ksoftirqd, hotcpu) = p;
 743                 break;
 744         case CPU_ONLINE:
 745         case CPU_ONLINE_FROZEN:
 746                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 747                 break;
 748 #ifdef CONFIG_HOTPLUG_CPU
 749         case CPU_UP_CANCELED:
 750         case CPU_UP_CANCELED_FROZEN:
 751                 if (!per_cpu(ksoftirqd, hotcpu))
 752                         break;
 753                 /* Unbind so it can run.  Fall thru. */
 754                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 755                              cpumask_any(cpu_online_mask));
 756         case CPU_DEAD:
 757         case CPU_DEAD_FROZEN: {
 758                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 759
 760                 p = per_cpu(ksoftirqd, hotcpu);
 761                 per_cpu(ksoftirqd, hotcpu) = NULL;
 762                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 763                 kthread_stop(p);
 764                 takeover_tasklets(hotcpu);
 765                 break;
 766         }
 767 #endif /* CONFIG_HOTPLUG_CPU */
 768         }
 769         return NOTIFY_OK;
 770 }
 771
 772 static struct notifier_block __cpuinitdata cpu_nfb = {
 773         .notifier_call = cpu_callback
 774 };
 775
 776 static __init int spawn_ksoftirqd(void)
 777 {
 778         void *cpu = (void *)(long)smp_processor_id();
 779         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 780
 781         BUG_ON(err == NOTIFY_BAD);
 782         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 783         register_cpu_notifier(&cpu_nfb);
 784         return 0;
 785 }
 786 early_initcall(spawn_ksoftirqd);
 787
 788 #ifdef CONFIG_SMP
 789 /*
 790  * Call a function on all processors
 791  */
 792 int on_each_cpu(void (*func) (void *info), void *info, int wait)
 793 {
 794         int ret = 0;
 795
 796         preempt_disable();
 797         ret = smp_call_function(func, info, wait);
 798         local_irq_disable();
 799         func(info);
 800         local_irq_enable();
 801         preempt_enable();
 802         return ret;
 803 }
 804 EXPORT_SYMBOL(on_each_cpu);
 805 #endif
 806
 807 /*
 808  * [ These __weak aliases are kept in a separate compilation unit, so that
 809  *   GCC does not inline them incorrectly. ]
 810  */
 811
 812 int __init __weak early_irq_init(void)
 813 {
 814         return 0;
 815 }
 816
 817 int __init __weak arch_probe_nr_irqs(void)
 818 {
 819         return 0;
 820 }
 821
 822 int __init __weak arch_early_irq_init(void)
 823 {
 824         return 0;
 825 }
 826
 827 int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
 828 {
 829         return 0;
 830 }