X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=kernel%2Ftime%2Ftick-sched.c;h=49e12f6a4bab8229e5a4aaed7a2a818163dc5432;hb=b10db7f0d2b589a7f88dc3026e150756cb437a28;hp=f4fc867f467d71b8dae84bf2e3f0fffbab955c64;hpb=d3ed782458f315c30ea679b919a2cc59f2b82565;p=safe%2Fjmp%2Flinux-2.6 diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index f4fc867..49e12f6 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -9,7 +9,7 @@ * * Started by: Thomas Gleixner and Ingo Molnar * - * For licencing details see kernel-base/COPYING + * Distribute under GPLv2. */ #include #include @@ -133,6 +133,8 @@ void tick_nohz_update_jiffies(void) if (!ts->tick_stopped) return; + touch_softlockup_watchdog(); + cpu_clear(cpu, nohz_cpu_mask); now = ktime_get(); @@ -151,8 +153,10 @@ void tick_nohz_update_jiffies(void) void tick_nohz_stop_sched_tick(void) { unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; + unsigned long rt_jiffies; struct tick_sched *ts; ktime_t last_update, expires, now, delta; + struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; int cpu; local_irq_save(flags); @@ -160,6 +164,18 @@ void tick_nohz_stop_sched_tick(void) cpu = smp_processor_id(); ts = &per_cpu(tick_cpu_sched, cpu); + /* + * If this cpu is offline and it is the one which updates + * jiffies, then give up the assignment and let it be taken by + * the cpu which runs the tick timer next. If we don't drop + * this here the jiffies might be stale and do_timer() never + * invoked. + */ + if (unlikely(!cpu_online(cpu))) { + if (cpu == tick_do_timer_cpu) + tick_do_timer_cpu = -1; + } + if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) goto end; @@ -167,9 +183,15 @@ void tick_nohz_stop_sched_tick(void) goto end; cpu = smp_processor_id(); - if (unlikely(local_softirq_pending())) - printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", - local_softirq_pending()); + if (unlikely(local_softirq_pending())) { + static int ratelimit; + + if (ratelimit < 10) { + printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", + local_softirq_pending()); + ratelimit++; + } + } now = ktime_get(); /* @@ -195,6 +217,10 @@ void tick_nohz_stop_sched_tick(void) next_jiffies = get_next_timer_interrupt(last_jiffies); delta_jiffies = next_jiffies - last_jiffies; + rt_jiffies = rt_needs_cpu(cpu); + if (rt_jiffies && rt_jiffies < delta_jiffies) + delta_jiffies = rt_jiffies; + if (rcu_needs_cpu(cpu)) delta_jiffies = 1; /* @@ -217,6 +243,14 @@ void tick_nohz_stop_sched_tick(void) * the scheduler tick in nohz_restart_sched_tick. */ if (!ts->tick_stopped) { + if (select_nohz_load_balancer(1)) { + /* + * sched tick not stopped! + */ + cpu_clear(cpu, nohz_cpu_mask); + goto out; + } + ts->idle_tick = ts->sched_timer.expires; ts->tick_stopped = 1; ts->idle_jiffies = last_jiffies; @@ -233,6 +267,21 @@ void tick_nohz_stop_sched_tick(void) if (cpu == tick_do_timer_cpu) tick_do_timer_cpu = -1; + ts->idle_sleeps++; + + /* + * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that + * there is no timer pending or at least extremly far + * into the future (12 days for HZ=1000). In this case + * we simply stop the tick timer: + */ + if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) { + ts->idle_expires.tv64 = KTIME_MAX; + if (ts->nohz_mode == NOHZ_MODE_HIGHRES) + hrtimer_cancel(&ts->sched_timer); + goto out; + } + /* * calculate the expiry time for the next timer wheel * timer @@ -240,7 +289,6 @@ void tick_nohz_stop_sched_tick(void) expires = ktime_add_ns(last_update, tick_period.tv64 * delta_jiffies); ts->idle_expires = expires; - ts->idle_sleeps++; if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { hrtimer_start(&ts->sched_timer, expires, @@ -248,7 +296,7 @@ void tick_nohz_stop_sched_tick(void) /* Check, if the timer was already in the past */ if (hrtimer_active(&ts->sched_timer)) goto out; - } else if(!tick_program_event(expires, 0)) + } else if (!tick_program_event(expires, 0)) goto out; /* * We are past the event already. So we crossed a @@ -262,12 +310,25 @@ void tick_nohz_stop_sched_tick(void) out: ts->next_jiffies = next_jiffies; ts->last_jiffies = last_jiffies; + ts->sleep_length = ktime_sub(dev->next_event, now); end: local_irq_restore(flags); } /** - * nohz_restart_sched_tick - restart the idle tick from the idle task + * tick_nohz_get_sleep_length - return the length of the current sleep + * + * Called from power state control code with interrupts disabled + */ +ktime_t tick_nohz_get_sleep_length(void) +{ + struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); + + return ts->sleep_length; +} + +/** + * tick_nohz_restart_sched_tick - restart the idle tick from the idle task * * Restart the idle tick when the CPU is woken up from idle */ @@ -285,6 +346,7 @@ void tick_nohz_restart_sched_tick(void) now = ktime_get(); local_irq_disable(); + select_nohz_load_balancer(0); tick_do_update_jiffies64(now); cpu_clear(cpu, nohz_cpu_mask); @@ -445,14 +507,13 @@ static inline void tick_nohz_switch_to_nohz(void) { } */ #ifdef CONFIG_HIGH_RES_TIMERS /* - * We rearm the timer until we get disabled by the idle code + * We rearm the timer until we get disabled by the idle code. * Called with interrupts disabled and timer->base->cpu_base->lock held. */ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) { struct tick_sched *ts = container_of(timer, struct tick_sched, sched_timer); - struct hrtimer_cpu_base *base = timer->base->cpu_base; struct pt_regs *regs = get_irq_regs(); ktime_t now = ktime_get(); int cpu = smp_processor_id(); @@ -490,15 +551,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) touch_softlockup_watchdog(); ts->idle_jiffies++; } - /* - * update_process_times() might take tasklist_lock, hence - * drop the base lock. sched-tick hrtimers are per-CPU and - * never accessible by userspace APIs, so this is safe to do. - */ - spin_unlock(&base->lock); update_process_times(user_mode(regs)); profile_tick(CPU_PROFILING); - spin_lock(&base->lock); } /* Do not restart, when we are in the idle loop */ @@ -517,6 +571,7 @@ void tick_setup_sched_timer(void) { struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); ktime_t now = ktime_get(); + u64 offset; /* * Emulate tick processing via per-CPU hrtimers: @@ -525,8 +580,12 @@ void tick_setup_sched_timer(void) ts->sched_timer.function = tick_sched_timer; ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; - /* Get the next period */ + /* Get the next period (per cpu) */ ts->sched_timer.expires = tick_init_jiffy_update(); + offset = ktime_to_ns(tick_period) >> 1; + do_div(offset, num_possible_cpus()); + offset *= smp_processor_id(); + ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset); for (;;) { hrtimer_forward(&ts->sched_timer, now, tick_period);