sched: Complete buddy switches

[safe/jmp/linux-2.6] / kernel / sched_fair.c

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 2ff850f..4f6356e 100644 (file)
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -24,7 +24,7 @@
 
 /*
  * Targeted preemption latency for CPU-bound tasks:
- * (default: 20ms * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 5ms * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * NOTE: this latency value is not the same as the concept of
  * 'timeslice length' - timeslices in CFS are of variable length
@@ -34,13 +34,13 @@
  * (to see the precise effective timeslice length of your workload,
  *  run vmstat and monitor the context-switches (cs) field)
  */
-unsigned int sysctl_sched_latency = 20000000ULL;
+unsigned int sysctl_sched_latency = 5000000ULL;
 
 /*
  * Minimal preemption granularity for CPU-bound tasks:
- * (default: 4 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
  */
-unsigned int sysctl_sched_min_granularity = 4000000ULL;
+unsigned int sysctl_sched_min_granularity = 1000000ULL;
 
 /*
  * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
@@ -48,10 +48,10 @@ unsigned int sysctl_sched_min_granularity = 4000000ULL;
 static unsigned int sched_nr_latency = 5;
 
 /*
- * After fork, child runs first. (default) If set to 0 then
+ * After fork, child runs first. If set to 0 (default) then
  * parent will (try to) run first.
  */
-const_debug unsigned int sysctl_sched_child_runs_first = 1;
+unsigned int sysctl_sched_child_runs_first __read_mostly;
 
 /*
  * sys_sched_yield() compat mode
@@ -63,13 +63,13 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
 
 /*
  * SCHED_OTHER wake-up granularity.
- * (default: 5 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-unsigned int sysctl_sched_wakeup_granularity = 5000000UL;
+unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -545,14 +545,13 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
        schedstat_set(se->wait_count, se->wait_count + 1);
        schedstat_set(se->wait_sum, se->wait_sum +
                        rq_of(cfs_rq)->clock - se->wait_start);
-       schedstat_set(se->wait_start, 0);
-
 #ifdef CONFIG_SCHEDSTATS
        if (entity_is_task(se)) {
                trace_sched_stat_wait(task_of(se),
                        rq_of(cfs_rq)->clock - se->wait_start);
        }
 #endif
+       schedstat_set(se->wait_start, 0);
 }
 
 static inline void
@@ -728,11 +727,11 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
 
                        vruntime -= thresh;
                }
-
-               /* ensure we never gain time by being placed backwards. */
-               vruntime = max_vruntime(se->vruntime, vruntime);
        }
 
+       /* ensure we never gain time by being placed backwards. */
+       vruntime = max_vruntime(se->vruntime, vruntime);
+
        se->vruntime = vruntime;
 }
 
@@ -1262,7 +1261,17 @@ wake_affine(struct sched_domain *this_sd, struct rq *this_rq,
        tg = task_group(p);
        weight = p->se.load.weight;
 
-       balanced = 100*(tl + effective_load(tg, this_cpu, weight, weight)) <=
+       /*
+        * In low-load situations, where prev_cpu is idle and this_cpu is idle
+        * due to the sync cause above having dropped tl to 0, we'll always have
+        * an imbalance, but there's really nothing you can do about that, so
+        * that's good too.
+        *
+        * Otherwise check if either cpus are near enough in load to allow this
+        * task to be woken on this_cpu.
+        */
+       balanced = !tl ||
+               100*(tl + effective_load(tg, this_cpu, weight, weight)) <=
                imbalance*(load + effective_load(tg, prev_cpu, 0, weight));
 
        /*
@@ -1305,8 +1314,6 @@ static int select_task_rq_fair(struct task_struct *p, int sync)
        this_rq         = cpu_rq(this_cpu);
        new_cpu         = prev_cpu;
 
-       if (prev_cpu == this_cpu)
-               goto out;
        /*
         * 'this_sd' is the first domain that both
         * this_cpu and prev_cpu are present in:
@@ -1494,7 +1501,8 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
         */
        if (sched_feat(LAST_BUDDY) && likely(se->on_rq && curr != rq->idle))
                set_last_buddy(se);
-       set_next_buddy(pse);
+       if (sched_feat(NEXT_BUDDY))
+               set_next_buddy(pse);
 
        /*
         * We can come here with TIF_NEED_RESCHED already set from new task
@@ -1519,9 +1527,10 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
        if (!sched_feat(WAKEUP_PREEMPT))
                return;
 
-       if (sched_feat(WAKEUP_OVERLAP) && (sync ||
-                       (se->avg_overlap < sysctl_sched_migration_cost &&
-                        pse->avg_overlap < sysctl_sched_migration_cost))) {
+       if ((sched_feat(WAKEUP_SYNC) && sync) ||
+           (sched_feat(WAKEUP_OVERLAP) &&
+            (se->avg_overlap < sysctl_sched_migration_cost &&
+             pse->avg_overlap < sysctl_sched_migration_cost))) {
                resched_task(curr);
                return;
        }
@@ -1748,6 +1757,8 @@ static void task_new_fair(struct rq *rq, struct task_struct *p)
        sched_info_queued(p);
 
        update_curr(cfs_rq);
+       if (curr)
+               se->vruntime = curr->vruntime;
        place_entity(cfs_rq, se, 1);
 
        /* 'curr' will be NULL if the child belongs to a different group */