sched_clock: fix NOHZ interaction

[safe/jmp/linux-2.6] / kernel / time / tick-sched.c

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 69dba0c..a87b046 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -48,6 +48,13 @@ static void tick_do_update_jiffies64(ktime_t now)
        unsigned long ticks = 0;
        ktime_t delta;
 
+       /*
+        * Do a quick check without holding xtime_lock:
+        */
+       delta = ktime_sub(now, last_jiffies_update);
+       if (delta.tv64 < tick_period.tv64)
+               return;
+
        /* Reevalute with xtime_lock held */
        write_seqlock(&xtime_lock);
 
@@ -133,8 +140,6 @@ void tick_nohz_update_jiffies(void)
        if (!ts->tick_stopped)
                return;
 
-       touch_softlockup_watchdog();
-
        cpu_clear(cpu, nohz_cpu_mask);
        now = ktime_get();
        ts->idle_waketime = now;
@@ -142,6 +147,8 @@ void tick_nohz_update_jiffies(void)
        local_irq_save(flags);
        tick_do_update_jiffies64(now);
        local_irq_restore(flags);
+
+       touch_softlockup_watchdog();
 }
 
 void tick_nohz_stop_idle(int cpu)
@@ -155,6 +162,8 @@ void tick_nohz_stop_idle(int cpu)
                ts->idle_lastupdate = now;
                ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
                ts->idle_active = 0;
+
+               sched_clock_idle_wakeup_event(0);
        }
 }
 
@@ -170,6 +179,7 @@ static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
        }
        ts->idle_entrytime = now;
        ts->idle_active = 1;
+       sched_clock_idle_sleep_event();
        return now;
 }
 
@@ -188,10 +198,9 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
  * Called either from the idle loop or from irq_exit() when an idle period was
  * just interrupted by an interrupt which did not cause a reschedule.
  */
-void tick_nohz_stop_sched_tick(void)
+void tick_nohz_stop_sched_tick(int inidle)
 {
        unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
-       unsigned long rt_jiffies;
        struct tick_sched *ts;
        ktime_t last_update, expires, now;
        struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
@@ -218,6 +227,11 @@ void tick_nohz_stop_sched_tick(void)
        if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
                goto end;
 
+       if (!inidle && !ts->inidle)
+               goto end;
+
+       ts->inidle = 1;
+
        if (need_resched())
                goto end;
 
@@ -229,6 +243,7 @@ void tick_nohz_stop_sched_tick(void)
                               local_softirq_pending());
                        ratelimit++;
                }
+               goto end;
        }
 
        ts->idle_calls++;
@@ -243,10 +258,6 @@ void tick_nohz_stop_sched_tick(void)
        next_jiffies = get_next_timer_interrupt(last_jiffies);
        delta_jiffies = next_jiffies - last_jiffies;
 
-       rt_jiffies = rt_needs_cpu(cpu);
-       if (rt_jiffies && rt_jiffies < delta_jiffies)
-               delta_jiffies = rt_jiffies;
-
        if (rcu_needs_cpu(cpu))
                delta_jiffies = 1;
        /*
@@ -369,11 +380,14 @@ void tick_nohz_restart_sched_tick(void)
        local_irq_disable();
        tick_nohz_stop_idle(cpu);
 
-       if (!ts->tick_stopped) {
+       if (!ts->inidle || !ts->tick_stopped) {
+               ts->inidle = 0;
                local_irq_enable();
                return;
        }
 
+       ts->inidle = 0;
+
        rcu_exit_nohz();
 
        /* Update jiffies first */
@@ -398,6 +412,7 @@ void tick_nohz_restart_sched_tick(void)
                sub_preempt_count(HARDIRQ_OFFSET);
        }
 
+       touch_softlockup_watchdog();
        /*
         * Cancel the scheduled timer and restore the tick
         */
@@ -631,17 +646,21 @@ void tick_setup_sched_timer(void)
                ts->nohz_mode = NOHZ_MODE_HIGHRES;
 #endif
 }
+#endif /* HIGH_RES_TIMERS */
 
+#if defined CONFIG_NO_HZ || defined CONFIG_HIGH_RES_TIMERS
 void tick_cancel_sched_timer(int cpu)
 {
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 
+# ifdef CONFIG_HIGH_RES_TIMERS
        if (ts->sched_timer.base)
                hrtimer_cancel(&ts->sched_timer);
+# endif
 
        ts->nohz_mode = NOHZ_MODE_INACTIVE;
 }
-#endif /* HIGH_RES_TIMERS */
+#endif
 
 /**
  * Async notification about clocksource changes