sched: Convert pi_lock to raw_spinlock

Convert locks which cannot be sleeping locks in preempt-rt to
raw_spinlocks.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Acked-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 8d10aa7..abec69b 100644 (file)
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -165,7 +165,7 @@ extern struct cred init_cred;
        .journal_info   = NULL,                                         \
        .cpu_timers     = INIT_CPU_TIMERS(tsk.cpu_timers),              \
        .fs_excl        = ATOMIC_INIT(0),                               \
-       .pi_lock        = __SPIN_LOCK_UNLOCKED(tsk.pi_lock),            \
+       .pi_lock        = __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock),        \
        .timer_slack_ns = 50000, /* 50 usec default slack */            \
        .pids = {                                                       \
                [PIDTYPE_PID]  = INIT_PID_LINK(PIDTYPE_PID),            \

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 294eb2f..41a9ea3 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1409,7 +1409,7 @@ struct task_struct {
 #endif
 
        /* Protection of the PI data structures: */
-       spinlock_t pi_lock;
+       raw_spinlock_t pi_lock;
 
 #ifdef CONFIG_RT_MUTEXES
        /* PI waiters blocked on a rt_mutex held by this task */

diff --git a/kernel/exit.c b/kernel/exit.c
index 6f50ef5..5962d7c 100644 (file)
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -933,7 +933,7 @@ NORET_TYPE void do_exit(long code)
         * an exiting task cleaning up the robust pi futexes.
         */
        smp_mb();
-       spin_unlock_wait(&tsk->pi_lock);
+       raw_spin_unlock_wait(&tsk->pi_lock);
 
        if (unlikely(in_atomic()))
                printk(KERN_INFO "note: %s[%d] exited with preempt_count %d\n",

diff --git a/kernel/fork.c b/kernel/fork.c
index 1415dc4..9bd9144 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -939,9 +939,9 @@ SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
 
 static void rt_mutex_init_task(struct task_struct *p)
 {
-       spin_lock_init(&p->pi_lock);
+       raw_spin_lock_init(&p->pi_lock);
 #ifdef CONFIG_RT_MUTEXES
-       plist_head_init(&p->pi_waiters, &p->pi_lock);
+       plist_head_init_raw(&p->pi_waiters, &p->pi_lock);
        p->pi_blocked_on = NULL;
 #endif
 }

diff --git a/kernel/futex.c b/kernel/futex.c
index 6af474d..320b369 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -403,9 +403,9 @@ static void free_pi_state(struct futex_pi_state *pi_state)
         * and has cleaned up the pi_state already
         */
        if (pi_state->owner) {
-               spin_lock_irq(&pi_state->owner->pi_lock);
+               raw_spin_lock_irq(&pi_state->owner->pi_lock);
                list_del_init(&pi_state->list);
-               spin_unlock_irq(&pi_state->owner->pi_lock);
+               raw_spin_unlock_irq(&pi_state->owner->pi_lock);
 
                rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner);
        }
@@ -470,18 +470,18 @@ void exit_pi_state_list(struct task_struct *curr)
         * pi_state_list anymore, but we have to be careful
         * versus waiters unqueueing themselves:
         */
-       spin_lock_irq(&curr->pi_lock);
+       raw_spin_lock_irq(&curr->pi_lock);
        while (!list_empty(head)) {
 
                next = head->next;
                pi_state = list_entry(next, struct futex_pi_state, list);
                key = pi_state->key;
                hb = hash_futex(&key);
-               spin_unlock_irq(&curr->pi_lock);
+               raw_spin_unlock_irq(&curr->pi_lock);
 
                spin_lock(&hb->lock);
 
-               spin_lock_irq(&curr->pi_lock);
+               raw_spin_lock_irq(&curr->pi_lock);
                /*
                 * We dropped the pi-lock, so re-check whether this
                 * task still owns the PI-state:
@@ -495,15 +495,15 @@ void exit_pi_state_list(struct task_struct *curr)
                WARN_ON(list_empty(&pi_state->list));
                list_del_init(&pi_state->list);
                pi_state->owner = NULL;
-               spin_unlock_irq(&curr->pi_lock);
+               raw_spin_unlock_irq(&curr->pi_lock);
 
                rt_mutex_unlock(&pi_state->pi_mutex);
 
                spin_unlock(&hb->lock);
 
-               spin_lock_irq(&curr->pi_lock);
+               raw_spin_lock_irq(&curr->pi_lock);
        }
-       spin_unlock_irq(&curr->pi_lock);
+       raw_spin_unlock_irq(&curr->pi_lock);
 }
 
 static int
@@ -558,7 +558,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
         * change of the task flags, we do this protected by
         * p->pi_lock:
         */
-       spin_lock_irq(&p->pi_lock);
+       raw_spin_lock_irq(&p->pi_lock);
        if (unlikely(p->flags & PF_EXITING)) {
                /*
                 * The task is on the way out. When PF_EXITPIDONE is
@@ -567,7 +567,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
                 */
                int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN;
 
-               spin_unlock_irq(&p->pi_lock);
+               raw_spin_unlock_irq(&p->pi_lock);
                put_task_struct(p);
                return ret;
        }
@@ -586,7 +586,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
        WARN_ON(!list_empty(&pi_state->list));
        list_add(&pi_state->list, &p->pi_state_list);
        pi_state->owner = p;
-       spin_unlock_irq(&p->pi_lock);
+       raw_spin_unlock_irq(&p->pi_lock);
 
        put_task_struct(p);
 
@@ -794,16 +794,16 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
                }
        }
 
-       spin_lock_irq(&pi_state->owner->pi_lock);
+       raw_spin_lock_irq(&pi_state->owner->pi_lock);
        WARN_ON(list_empty(&pi_state->list));
        list_del_init(&pi_state->list);
-       spin_unlock_irq(&pi_state->owner->pi_lock);
+       raw_spin_unlock_irq(&pi_state->owner->pi_lock);
 
-       spin_lock_irq(&new_owner->pi_lock);
+       raw_spin_lock_irq(&new_owner->pi_lock);
        WARN_ON(!list_empty(&pi_state->list));
        list_add(&pi_state->list, &new_owner->pi_state_list);
        pi_state->owner = new_owner;
-       spin_unlock_irq(&new_owner->pi_lock);
+       raw_spin_unlock_irq(&new_owner->pi_lock);
 
        spin_unlock(&pi_state->pi_mutex.wait_lock);
        rt_mutex_unlock(&pi_state->pi_mutex);
@@ -1529,18 +1529,18 @@ retry:
         * itself.
         */
        if (pi_state->owner != NULL) {
-               spin_lock_irq(&pi_state->owner->pi_lock);
+               raw_spin_lock_irq(&pi_state->owner->pi_lock);
                WARN_ON(list_empty(&pi_state->list));
                list_del_init(&pi_state->list);
-               spin_unlock_irq(&pi_state->owner->pi_lock);
+               raw_spin_unlock_irq(&pi_state->owner->pi_lock);
        }
 
        pi_state->owner = newowner;
 
-       spin_lock_irq(&newowner->pi_lock);
+       raw_spin_lock_irq(&newowner->pi_lock);
        WARN_ON(!list_empty(&pi_state->list));
        list_add(&pi_state->list, &newowner->pi_state_list);
-       spin_unlock_irq(&newowner->pi_lock);
+       raw_spin_unlock_irq(&newowner->pi_lock);
        return 0;
 
        /*

diff --git a/kernel/rtmutex-debug.c b/kernel/rtmutex-debug.c
index 5fcb4fe..ddabb54 100644 (file)
--- a/kernel/rtmutex-debug.c
+++ b/kernel/rtmutex-debug.c
@@ -37,8 +37,8 @@ do {                                                          \
        if (rt_trace_on) {                                      \
                rt_trace_on = 0;                                \
                console_verbose();                              \
-               if (spin_is_locked(&current->pi_lock))          \
-                       spin_unlock(&current->pi_lock);         \
+               if (raw_spin_is_locked(&current->pi_lock))      \
+                       raw_spin_unlock(&current->pi_lock);     \
        }                                                       \
 } while (0)
 

diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 29bd4ba..d33da47 100644 (file)
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -138,9 +138,9 @@ static void rt_mutex_adjust_prio(struct task_struct *task)
 {
        unsigned long flags;
 
-       spin_lock_irqsave(&task->pi_lock, flags);
+       raw_spin_lock_irqsave(&task->pi_lock, flags);
        __rt_mutex_adjust_prio(task);
-       spin_unlock_irqrestore(&task->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 }
 
 /*
@@ -195,7 +195,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
        /*
         * Task can not go away as we did a get_task() before !
         */
-       spin_lock_irqsave(&task->pi_lock, flags);
+       raw_spin_lock_irqsave(&task->pi_lock, flags);
 
        waiter = task->pi_blocked_on;
        /*
@@ -232,7 +232,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
 
        lock = waiter->lock;
        if (!spin_trylock(&lock->wait_lock)) {
-               spin_unlock_irqrestore(&task->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&task->pi_lock, flags);
                cpu_relax();
                goto retry;
        }
@@ -253,13 +253,13 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
        plist_add(&waiter->list_entry, &lock->wait_list);
 
        /* Release the task */
-       spin_unlock_irqrestore(&task->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&task->pi_lock, flags);
        put_task_struct(task);
 
        /* Grab the next task */
        task = rt_mutex_owner(lock);
        get_task_struct(task);
-       spin_lock_irqsave(&task->pi_lock, flags);
+       raw_spin_lock_irqsave(&task->pi_lock, flags);
 
        if (waiter == rt_mutex_top_waiter(lock)) {
                /* Boost the owner */
@@ -277,7 +277,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
                __rt_mutex_adjust_prio(task);
        }
 
-       spin_unlock_irqrestore(&task->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 
        top_waiter = rt_mutex_top_waiter(lock);
        spin_unlock(&lock->wait_lock);
@@ -288,7 +288,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
        goto again;
 
  out_unlock_pi:
-       spin_unlock_irqrestore(&task->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  out_put_task:
        put_task_struct(task);
 
@@ -313,9 +313,9 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
        if (pendowner == task)
                return 1;
 
-       spin_lock_irqsave(&pendowner->pi_lock, flags);
+       raw_spin_lock_irqsave(&pendowner->pi_lock, flags);
        if (task->prio >= pendowner->prio) {
-               spin_unlock_irqrestore(&pendowner->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
                return 0;
        }
 
@@ -325,7 +325,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
         * priority.
         */
        if (likely(!rt_mutex_has_waiters(lock))) {
-               spin_unlock_irqrestore(&pendowner->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
                return 1;
        }
 
@@ -333,7 +333,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
        next = rt_mutex_top_waiter(lock);
        plist_del(&next->pi_list_entry, &pendowner->pi_waiters);
        __rt_mutex_adjust_prio(pendowner);
-       spin_unlock_irqrestore(&pendowner->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
 
        /*
         * We are going to steal the lock and a waiter was
@@ -350,10 +350,10 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
         * might be task:
         */
        if (likely(next->task != task)) {
-               spin_lock_irqsave(&task->pi_lock, flags);
+               raw_spin_lock_irqsave(&task->pi_lock, flags);
                plist_add(&next->pi_list_entry, &task->pi_waiters);
                __rt_mutex_adjust_prio(task);
-               spin_unlock_irqrestore(&task->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&task->pi_lock, flags);
        }
        return 1;
 }
@@ -420,7 +420,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
        unsigned long flags;
        int chain_walk = 0, res;
 
-       spin_lock_irqsave(&task->pi_lock, flags);
+       raw_spin_lock_irqsave(&task->pi_lock, flags);
        __rt_mutex_adjust_prio(task);
        waiter->task = task;
        waiter->lock = lock;
@@ -434,17 +434,17 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
 
        task->pi_blocked_on = waiter;
 
-       spin_unlock_irqrestore(&task->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 
        if (waiter == rt_mutex_top_waiter(lock)) {
-               spin_lock_irqsave(&owner->pi_lock, flags);
+               raw_spin_lock_irqsave(&owner->pi_lock, flags);
                plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
                plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
 
                __rt_mutex_adjust_prio(owner);
                if (owner->pi_blocked_on)
                        chain_walk = 1;
-               spin_unlock_irqrestore(&owner->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
        }
        else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
                chain_walk = 1;
@@ -483,7 +483,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
        struct task_struct *pendowner;
        unsigned long flags;
 
-       spin_lock_irqsave(&current->pi_lock, flags);
+       raw_spin_lock_irqsave(&current->pi_lock, flags);
 
        waiter = rt_mutex_top_waiter(lock);
        plist_del(&waiter->list_entry, &lock->wait_list);
@@ -500,7 +500,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
 
        rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING);
 
-       spin_unlock_irqrestore(&current->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&current->pi_lock, flags);
 
        /*
         * Clear the pi_blocked_on variable and enqueue a possible
@@ -509,7 +509,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
         * waiter with higher priority than pending-owner->normal_prio
         * is blocked on the unboosted (pending) owner.
         */
-       spin_lock_irqsave(&pendowner->pi_lock, flags);
+       raw_spin_lock_irqsave(&pendowner->pi_lock, flags);
 
        WARN_ON(!pendowner->pi_blocked_on);
        WARN_ON(pendowner->pi_blocked_on != waiter);
@@ -523,7 +523,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
                next = rt_mutex_top_waiter(lock);
                plist_add(&next->pi_list_entry, &pendowner->pi_waiters);
        }
-       spin_unlock_irqrestore(&pendowner->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
 
        wake_up_process(pendowner);
 }
@@ -541,15 +541,15 @@ static void remove_waiter(struct rt_mutex *lock,
        unsigned long flags;
        int chain_walk = 0;
 
-       spin_lock_irqsave(&current->pi_lock, flags);
+       raw_spin_lock_irqsave(&current->pi_lock, flags);
        plist_del(&waiter->list_entry, &lock->wait_list);
        waiter->task = NULL;
        current->pi_blocked_on = NULL;
-       spin_unlock_irqrestore(&current->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&current->pi_lock, flags);
 
        if (first && owner != current) {
 
-               spin_lock_irqsave(&owner->pi_lock, flags);
+               raw_spin_lock_irqsave(&owner->pi_lock, flags);
 
                plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
 
@@ -564,7 +564,7 @@ static void remove_waiter(struct rt_mutex *lock,
                if (owner->pi_blocked_on)
                        chain_walk = 1;
 
-               spin_unlock_irqrestore(&owner->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
        }
 
        WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
@@ -592,15 +592,15 @@ void rt_mutex_adjust_pi(struct task_struct *task)
        struct rt_mutex_waiter *waiter;
        unsigned long flags;
 
-       spin_lock_irqsave(&task->pi_lock, flags);
+       raw_spin_lock_irqsave(&task->pi_lock, flags);
 
        waiter = task->pi_blocked_on;
        if (!waiter || waiter->list_entry.prio == task->prio) {
-               spin_unlock_irqrestore(&task->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&task->pi_lock, flags);
                return;
        }
 
-       spin_unlock_irqrestore(&task->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 
        /* gets dropped in rt_mutex_adjust_prio_chain()! */
        get_task_struct(task);

diff --git a/kernel/sched.c b/kernel/sched.c
index 01c5016..18cceee 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -6323,7 +6323,7 @@ recheck:
         * make sure no PI-waiters arrive (or leave) while we are
         * changing the priority of the task:
         */
-       spin_lock_irqsave(&p->pi_lock, flags);
+       raw_spin_lock_irqsave(&p->pi_lock, flags);
        /*
         * To be able to change p->policy safely, the apropriate
         * runqueue lock must be held.
@@ -6333,7 +6333,7 @@ recheck:
        if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
                policy = oldpolicy = -1;
                __task_rq_unlock(rq);
-               spin_unlock_irqrestore(&p->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&p->pi_lock, flags);
                goto recheck;
        }
        update_rq_clock(rq);
@@ -6357,7 +6357,7 @@ recheck:
                check_class_changed(rq, p, prev_class, oldprio, running);
        }
        __task_rq_unlock(rq);
-       spin_unlock_irqrestore(&p->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 
        rt_mutex_adjust_pi(p);
 
@@ -9624,7 +9624,7 @@ void __init sched_init(void)
 #endif
 
 #ifdef CONFIG_RT_MUTEXES
-       plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
+       plist_head_init_raw(&init_task.pi_waiters, &init_task.pi_lock);
 #endif
 
        /*
@@ -9749,13 +9749,13 @@ void normalize_rt_tasks(void)
                        continue;
                }
 
-               spin_lock(&p->pi_lock);
+               raw_spin_lock(&p->pi_lock);
                rq = __task_rq_lock(p);
 
                normalize_task(rq, p);
 
                __task_rq_unlock(rq);
-               spin_unlock(&p->pi_lock);
+               raw_spin_unlock(&p->pi_lock);
        } while_each_thread(g, p);
 
        read_unlock_irqrestore(&tasklist_lock, flags);