* Thomas Gleixner, Mike Kravetz
*/
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
* We should never call set_task_cpu() on a blocked task,
* ttwu() will sort out the placement.
*/
- WARN_ON(p->state != TASK_RUNNING && p->state != TASK_WAKING);
+ WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
+ !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
#endif
trace_sched_migrate_task(p, new_cpu);
- if (task_cpu(p) == new_cpu)
- return;
-
- p->se.nr_migrations++;
- perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
+ if (task_cpu(p) != new_cpu) {
+ p->se.nr_migrations++;
+ perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
+ }
__set_task_cpu(p, new_cpu);
}
}
/*
- * Called from:
- *
- * - fork, @p is stable because it isn't on the tasklist yet
- *
- * - exec, @p is unstable, retry loop
+ * Gets called from 3 sites (exec, fork, wakeup), since it is called without
+ * holding rq->lock we need to ensure ->cpus_allowed is stable, this is done
+ * by:
*
- * - wake-up, we serialize ->cpus_allowed against TASK_WAKING so
- * we should be good.
+ * exec: is unstable, retry loop
+ * fork & wake-up: serialize ->cpus_allowed against TASK_WAKING
*/
static inline
int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
* not worry about this generic constraint ]
*/
if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
- !cpu_active(cpu)))
+ !cpu_online(cpu)))
cpu = select_fallback_rq(task_cpu(p), p);
return cpu;
if (p->sched_class->task_fork)
p->sched_class->task_fork(p);
-#ifdef CONFIG_SMP
- cpu = select_task_rq(p, SD_BALANCE_FORK, 0);
-#endif
set_task_cpu(p, cpu);
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
{
unsigned long flags;
struct rq *rq;
+ int cpu = get_cpu();
+
+#ifdef CONFIG_SMP
+ /*
+ * Fork balancing, do it here and not earlier because:
+ * - cpus_allowed can change in the fork path
+ * - any previously selected cpu might disappear through hotplug
+ *
+ * We still have TASK_WAKING but PF_STARTING is gone now, meaning
+ * ->cpus_allowed is stable, we have preemption disabled, meaning
+ * cpu_online_mask is stable.
+ */
+ cpu = select_task_rq(p, SD_BALANCE_FORK, 0);
+ set_task_cpu(p, cpu);
+#endif
rq = task_rq_lock(p, &flags);
BUG_ON(p->state != TASK_WAKING);
p->sched_class->task_woken(rq, p);
#endif
task_rq_unlock(rq, &flags);
+ put_cpu();
}
#ifdef CONFIG_PREEMPT_NOTIFIERS
{
struct pt_regs *regs = get_irq_regs();
- pr_err("BUG: scheduling while atomic: %s/%d/0x%08x\n",
- prev->comm, prev->pid, preempt_count());
+ printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
+ prev->comm, prev->pid, preempt_count());
debug_show_held_locks(prev);
print_modules();
post_schedule(rq);
- if (unlikely(reacquire_kernel_lock(current) < 0))
+ if (unlikely(reacquire_kernel_lock(current) < 0)) {
+ prev = rq->curr;
+ switch_count = &prev->nivcsw;
goto need_resched_nonpreemptible;
+ }
preempt_enable_no_resched();
if (need_resched())
unsigned state;
state = p->state ? __ffs(p->state) + 1 : 0;
- pr_info("%-13.13s %c", p->comm,
+ printk(KERN_INFO "%-13.13s %c", p->comm,
state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
#if BITS_PER_LONG == 32
if (state == TASK_RUNNING)
- pr_cont(" running ");
+ printk(KERN_CONT " running ");
else
- pr_cont(" %08lx ", thread_saved_pc(p));
+ printk(KERN_CONT " %08lx ", thread_saved_pc(p));
#else
if (state == TASK_RUNNING)
- pr_cont(" running task ");
+ printk(KERN_CONT " running task ");
else
- pr_cont(" %016lx ", thread_saved_pc(p));
+ printk(KERN_CONT " %016lx ", thread_saved_pc(p));
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
free = stack_not_used(p);
#endif
- pr_cont("%5lu %5d %6d 0x%08lx\n", free,
+ printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
task_pid_nr(p), task_pid_nr(p->real_parent),
(unsigned long)task_thread_info(p)->flags);
struct task_struct *g, *p;
#if BITS_PER_LONG == 32
- pr_info(" task PC stack pid father\n");
+ printk(KERN_INFO
+ " task PC stack pid father\n");
#else
- pr_info(" task PC stack pid father\n");
+ printk(KERN_INFO
+ " task PC stack pid father\n");
#endif
read_lock(&tasklist_lock);
do_each_thread(g, p) {
* the ->cpus_allowed mask from under waking tasks, which would be
* possible when we change rq->lock in ttwu(), so synchronize against
* TASK_WAKING to avoid that.
+ *
+ * Make an exception for freshly cloned tasks, since cpuset namespaces
+ * might move the task about, we have to validate the target in
+ * wake_up_new_task() anyway since the cpu might have gone away.
*/
again:
- while (p->state == TASK_WAKING)
+ while (p->state == TASK_WAKING && !(p->flags & PF_STARTING))
cpu_relax();
rq = task_rq_lock(p, &flags);
- if (p->state == TASK_WAKING) {
+ if (p->state == TASK_WAKING && !(p->flags & PF_STARTING)) {
task_rq_unlock(rq, &flags);
goto again;
}
printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
if (!(sd->flags & SD_LOAD_BALANCE)) {
- pr_cont("does not load-balance\n");
+ printk("does not load-balance\n");
if (sd->parent)
- pr_err("ERROR: !SD_LOAD_BALANCE domain has parent\n");
+ printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
+ " has parent");
return -1;
}
- pr_cont("span %s level %s\n", str, sd->name);
+ printk(KERN_CONT "span %s level %s\n", str, sd->name);
if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
- pr_err("ERROR: domain->span does not contain CPU%d\n", cpu);
+ printk(KERN_ERR "ERROR: domain->span does not contain "
+ "CPU%d\n", cpu);
}
if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
- pr_err("ERROR: domain->groups does not contain CPU%d\n", cpu);
+ printk(KERN_ERR "ERROR: domain->groups does not contain"
+ " CPU%d\n", cpu);
}
printk(KERN_DEBUG "%*s groups:", level + 1, "");
do {
if (!group) {
- pr_cont("\n");
- pr_err("ERROR: group is NULL\n");
+ printk("\n");
+ printk(KERN_ERR "ERROR: group is NULL\n");
break;
}
if (!group->cpu_power) {
- pr_cont("\n");
- pr_err("ERROR: domain->cpu_power not set\n");
+ printk(KERN_CONT "\n");
+ printk(KERN_ERR "ERROR: domain->cpu_power not "
+ "set\n");
break;
}
if (!cpumask_weight(sched_group_cpus(group))) {
- pr_cont("\n");
- pr_err("ERROR: empty group\n");
+ printk(KERN_CONT "\n");
+ printk(KERN_ERR "ERROR: empty group\n");
break;
}
if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
- pr_cont("\n");
- pr_err("ERROR: repeated CPUs\n");
+ printk(KERN_CONT "\n");
+ printk(KERN_ERR "ERROR: repeated CPUs\n");
break;
}
cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
- pr_cont(" %s", str);
+ printk(KERN_CONT " %s", str);
if (group->cpu_power != SCHED_LOAD_SCALE) {
- pr_cont(" (cpu_power = %d)", group->cpu_power);
+ printk(KERN_CONT " (cpu_power = %d)",
+ group->cpu_power);
}
group = group->next;
} while (group != sd->groups);
- pr_cont("\n");
+ printk(KERN_CONT "\n");
if (!cpumask_equal(sched_domain_span(sd), groupmask))
- pr_err("ERROR: groups don't span domain->span\n");
+ printk(KERN_ERR "ERROR: groups don't span domain->span\n");
if (sd->parent &&
!cpumask_subset(groupmask, sched_domain_span(sd->parent)))
- pr_err("ERROR: parent span is not a superset of domain->span\n");
+ printk(KERN_ERR "ERROR: parent span is not a superset "
+ "of domain->span\n");
return 0;
}
sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
GFP_KERNEL, num);
if (!sg) {
- pr_warning("Can not alloc domain group for node %d\n", num);
+ printk(KERN_WARNING "Can not alloc domain group for node %d\n",
+ num);
return -ENOMEM;
}
d->sched_group_nodes[num] = sg;
sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
GFP_KERNEL, num);
if (!sg) {
- pr_warning("Can not alloc domain group for node %d\n",
- j);
+ printk(KERN_WARNING
+ "Can not alloc domain group for node %d\n", j);
return -ENOMEM;
}
sg->cpu_power = 0;
d->sched_group_nodes = kcalloc(nr_node_ids,
sizeof(struct sched_group *), GFP_KERNEL);
if (!d->sched_group_nodes) {
- pr_warning("Can not alloc sched group node list\n");
+ printk(KERN_WARNING "Can not alloc sched group node list\n");
return sa_notcovered;
}
sched_group_nodes_bycpu[cpumask_first(cpu_map)] = d->sched_group_nodes;
return sa_send_covered;
d->rd = alloc_rootdomain();
if (!d->rd) {
- pr_warning("Cannot alloc root domain\n");
+ printk(KERN_WARNING "Cannot alloc root domain\n");
return sa_tmpmask;
}
return sa_rootdomain;
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
static inline int preempt_count_equals(int preempt_offset)
{
- int nested = preempt_count() & ~PREEMPT_ACTIVE;
+ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
}
return;
prev_jiffy = jiffies;
- pr_err("BUG: sleeping function called from invalid context at %s:%d\n",
- file, line);
- pr_err("in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
- in_atomic(), irqs_disabled(),
- current->pid, current->comm);
+ printk(KERN_ERR
+ "BUG: sleeping function called from invalid context at %s:%d\n",
+ file, line);
+ printk(KERN_ERR
+ "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
+ in_atomic(), irqs_disabled(),
+ current->pid, current->comm);
debug_show_held_locks(current);
if (irqs_disabled())