#include <linux/utsname.h>
#include <linux/hash.h>
#include <linux/ftrace.h>
+#include <linux/stringify.h>
#include <asm/sections.h>
#include "lockdep_internals.h"
+#define CREATE_TRACE_POINTS
+#include <trace/events/lockdep.h>
+
#ifdef CONFIG_PROVE_LOCKING
int prove_locking = 1;
module_param(prove_locking, int, 0644);
atomic_t nr_find_usage_forwards_recursions;
atomic_t nr_find_usage_backwards_checks;
atomic_t nr_find_usage_backwards_recursions;
-# define debug_atomic_inc(ptr) atomic_inc(ptr)
-# define debug_atomic_dec(ptr) atomic_dec(ptr)
-# define debug_atomic_read(ptr) atomic_read(ptr)
-#else
-# define debug_atomic_inc(ptr) do { } while (0)
-# define debug_atomic_dec(ptr) do { } while (0)
-# define debug_atomic_read(ptr) 0
#endif
/*
* Locking printouts:
*/
-#define __STR(foo) #foo
-#define STR(foo) __STR(foo)
-
#define __USAGE(__STATE) \
- [LOCK_USED_IN_##__STATE] = "IN-"STR(__STATE)"-W", \
- [LOCK_ENABLED_##__STATE] = STR(__STATE)"-ON-W", \
- [LOCK_USED_IN_##__STATE##_READ] = "IN-"STR(__STATE)"-R", \
- [LOCK_ENABLED_##__STATE##_READ] = STR(__STATE)"-ON-R",
+ [LOCK_USED_IN_##__STATE] = "IN-"__stringify(__STATE)"-W", \
+ [LOCK_ENABLED_##__STATE] = __stringify(__STATE)"-ON-W", \
+ [LOCK_USED_IN_##__STATE##_READ] = "IN-"__stringify(__STATE)"-R",\
+ [LOCK_ENABLED_##__STATE##_READ] = __stringify(__STATE)"-ON-R",
static const char *usage_str[] =
{
return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);
}
-void
-get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3,
- char *c4, char *c5, char *c6)
+static inline unsigned long lock_flag(enum lock_usage_bit bit)
{
- *c1 = '.', *c2 = '.', *c3 = '.', *c4 = '.', *c5 = '.', *c6 = '.';
-
- if (class->usage_mask & LOCKF_USED_IN_HARDIRQ)
- *c1 = '+';
- else
- if (class->usage_mask & LOCKF_ENABLED_HARDIRQ)
- *c1 = '-';
+ return 1UL << bit;
+}
- if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ)
- *c2 = '+';
- else
- if (class->usage_mask & LOCKF_ENABLED_SOFTIRQ)
- *c2 = '-';
+static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit)
+{
+ char c = '.';
- if (class->usage_mask & LOCKF_ENABLED_HARDIRQ_READ)
- *c3 = '-';
- if (class->usage_mask & LOCKF_USED_IN_HARDIRQ_READ) {
- *c3 = '+';
- if (class->usage_mask & LOCKF_ENABLED_HARDIRQ_READ)
- *c3 = '?';
+ if (class->usage_mask & lock_flag(bit + 2))
+ c = '+';
+ if (class->usage_mask & lock_flag(bit)) {
+ c = '-';
+ if (class->usage_mask & lock_flag(bit + 2))
+ c = '?';
}
- if (class->usage_mask & LOCKF_ENABLED_SOFTIRQ_READ)
- *c4 = '-';
- if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ_READ) {
- *c4 = '+';
- if (class->usage_mask & LOCKF_ENABLED_SOFTIRQ_READ)
- *c4 = '?';
- }
+ return c;
+}
- if (class->usage_mask & LOCKF_USED_IN_RECLAIM_FS)
- *c5 = '+';
- else
- if (class->usage_mask & LOCKF_ENABLED_RECLAIM_FS)
- *c5 = '-';
+void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS])
+{
+ int i = 0;
- if (class->usage_mask & LOCKF_ENABLED_RECLAIM_FS_READ)
- *c6 = '-';
- if (class->usage_mask & LOCKF_USED_IN_RECLAIM_FS_READ) {
- *c6 = '+';
- if (class->usage_mask & LOCKF_ENABLED_RECLAIM_FS_READ)
- *c6 = '?';
- }
+#define LOCKDEP_STATE(__STATE) \
+ usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE); \
+ usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);
+#include "lockdep_states.h"
+#undef LOCKDEP_STATE
+ usage[i] = '\0';
}
static void print_lock_name(struct lock_class *class)
{
- char str[KSYM_NAME_LEN], c1, c2, c3, c4, c5, c6;
+ char str[KSYM_NAME_LEN], usage[LOCK_USAGE_CHARS];
const char *name;
- get_usage_chars(class, &c1, &c2, &c3, &c4, &c5, &c6);
+ get_usage_chars(class, usage);
name = class->name;
if (!name) {
if (class->subclass)
printk("/%d", class->subclass);
}
- printk("){%c%c%c%c%c%c}", c1, c2, c3, c4, c5, c6);
+ printk("){%s}", usage);
}
static void print_lockdep_cache(struct lockdep_map *lock)
printk("BUG: MAX_LOCKDEP_KEYS too low!\n");
printk("turning off the locking correctness validator.\n");
+ dump_stack();
return NULL;
}
class = lock_classes + nr_lock_classes++;
printk("BUG: MAX_LOCKDEP_ENTRIES too low!\n");
printk("turning off the locking correctness validator.\n");
+ dump_stack();
return NULL;
}
return list_entries + nr_list_entries++;
bit_backwards, bit_forwards, irqclass);
}
-static int
-check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
- struct held_lock *next)
+static const char *state_names[] = {
+#define LOCKDEP_STATE(__STATE) \
+ __stringify(__STATE),
+#include "lockdep_states.h"
+#undef LOCKDEP_STATE
+};
+
+static const char *state_rnames[] = {
+#define LOCKDEP_STATE(__STATE) \
+ __stringify(__STATE)"-READ",
+#include "lockdep_states.h"
+#undef LOCKDEP_STATE
+};
+
+static inline const char *state_name(enum lock_usage_bit bit)
{
- /*
- * Prove that the new dependency does not connect a hardirq-safe
- * lock with a hardirq-unsafe lock - to achieve this we search
- * the backwards-subgraph starting at <prev>, and the
- * forwards-subgraph starting at <next>:
- */
- if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ,
- LOCK_ENABLED_HARDIRQ, "hard"))
- return 0;
+ return (bit & 1) ? state_rnames[bit >> 2] : state_names[bit >> 2];
+}
+static int exclusive_bit(int new_bit)
+{
/*
- * Prove that the new dependency does not connect a hardirq-safe-read
- * lock with a hardirq-unsafe lock - to achieve this we search
- * the backwards-subgraph starting at <prev>, and the
- * forwards-subgraph starting at <next>:
+ * USED_IN
+ * USED_IN_READ
+ * ENABLED
+ * ENABLED_READ
+ *
+ * bit 0 - write/read
+ * bit 1 - used_in/enabled
+ * bit 2+ state
*/
- if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ_READ,
- LOCK_ENABLED_HARDIRQ, "hard-read"))
- return 0;
+
+ int state = new_bit & ~3;
+ int dir = new_bit & 2;
/*
- * Prove that the new dependency does not connect a softirq-safe
- * lock with a softirq-unsafe lock - to achieve this we search
- * the backwards-subgraph starting at <prev>, and the
- * forwards-subgraph starting at <next>:
+ * keep state, bit flip the direction and strip read.
*/
- if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ,
- LOCK_ENABLED_SOFTIRQ, "soft"))
- return 0;
+ return state | (dir ^ 2);
+}
+
+static int check_irq_usage(struct task_struct *curr, struct held_lock *prev,
+ struct held_lock *next, enum lock_usage_bit bit)
+{
/*
- * Prove that the new dependency does not connect a softirq-safe-read
- * lock with a softirq-unsafe lock - to achieve this we search
+ * Prove that the new dependency does not connect a hardirq-safe
+ * lock with a hardirq-unsafe lock - to achieve this we search
* the backwards-subgraph starting at <prev>, and the
* forwards-subgraph starting at <next>:
*/
- if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ_READ,
- LOCK_ENABLED_SOFTIRQ, "soft"))
+ if (!check_usage(curr, prev, next, bit,
+ exclusive_bit(bit), state_name(bit)))
return 0;
+ bit++; /* _READ */
+
/*
- * Prove that the new dependency does not connect a reclaim-fs-safe
- * lock with a reclaim-fs-unsafe lock - to achieve this we search
+ * Prove that the new dependency does not connect a hardirq-safe-read
+ * lock with a hardirq-unsafe lock - to achieve this we search
* the backwards-subgraph starting at <prev>, and the
* forwards-subgraph starting at <next>:
*/
- if (!check_usage(curr, prev, next, LOCK_USED_IN_RECLAIM_FS,
- LOCK_ENABLED_RECLAIM_FS, "reclaim-fs"))
+ if (!check_usage(curr, prev, next, bit,
+ exclusive_bit(bit), state_name(bit)))
return 0;
- /*
- * Prove that the new dependency does not connect a reclaim-fs-safe-read
- * lock with a reclaim-fs-unsafe lock - to achieve this we search
- * the backwards-subgraph starting at <prev>, and the
- * forwards-subgraph starting at <next>:
- */
- if (!check_usage(curr, prev, next, LOCK_USED_IN_RECLAIM_FS_READ,
- LOCK_ENABLED_RECLAIM_FS, "reclaim-fs-read"))
+ return 1;
+}
+
+static int
+check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
+ struct held_lock *next)
+{
+#define LOCKDEP_STATE(__STATE) \
+ if (!check_irq_usage(curr, prev, next, LOCK_USED_IN_##__STATE)) \
return 0;
+#include "lockdep_states.h"
+#undef LOCKDEP_STATE
return 1;
}
printk("BUG: MAX_LOCKDEP_CHAINS too low!\n");
printk("turning off the locking correctness validator.\n");
+ dump_stack();
return 0;
}
chain = lock_chains + nr_lock_chains++;
curr->comm, task_pid_nr(curr));
print_lock(this);
if (forwards)
- printk("but this lock took another, %s-irq-unsafe lock in the past:\n", irqclass);
+ printk("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
else
- printk("but this lock was taken by another, %s-irq-safe lock in the past:\n", irqclass);
+ printk("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
print_lock_name(other);
printk("\n\nand interrupts could create inverse lock ordering between them.\n\n");
print_ip_sym(curr->softirq_disable_ip);
}
-static int hardirq_verbose(struct lock_class *class)
+static int HARDIRQ_verbose(struct lock_class *class)
{
#if HARDIRQ_VERBOSE
return class_filter(class);
return 0;
}
-static int softirq_verbose(struct lock_class *class)
+static int SOFTIRQ_verbose(struct lock_class *class)
{
#if SOFTIRQ_VERBOSE
return class_filter(class);
return 0;
}
-static int reclaim_verbose(struct lock_class *class)
+static int RECLAIM_FS_verbose(struct lock_class *class)
{
#if RECLAIM_VERBOSE
return class_filter(class);
#define STRICT_READ_CHECKS 1
-static const char *state_names[] = {
-#define LOCKDEP_STATE(__STATE) \
- STR(__STATE),
-#include "lockdep_states.h"
-#undef LOCKDEP_STATE
-};
-
-static inline const char *state_name(enum lock_usage_bit bit)
-{
- return state_names[bit >> 2];
-}
-
-static const char *state_rnames[] = {
+static int (*state_verbose_f[])(struct lock_class *class) = {
#define LOCKDEP_STATE(__STATE) \
- STR(__STATE)"-READ",
+ __STATE##_verbose,
#include "lockdep_states.h"
#undef LOCKDEP_STATE
};
-static inline const char *state_rname(enum lock_usage_bit bit)
+static inline int state_verbose(enum lock_usage_bit bit,
+ struct lock_class *class)
{
- return state_rnames[bit >> 2];
+ return state_verbose_f[bit >> 2](class);
}
-static int exclusive_bit(int new_bit)
-{
- /*
- * USED_IN
- * USED_IN_READ
- * ENABLED
- * ENABLED_READ
- *
- * bit 0 - write/read
- * bit 1 - used_in/enabled
- * bit 2+ state
- */
-
- int state = new_bit & ~3;
- int dir = new_bit & 2;
-
- return state | (dir ^ 2);
-}
+typedef int (*check_usage_f)(struct task_struct *, struct held_lock *,
+ enum lock_usage_bit bit, const char *name);
static int
-mark_lock_irq_used_in(struct task_struct *curr, struct held_lock *this,
- int new_bit,
- int (*verbose)(struct lock_class *class))
+mark_lock_irq(struct task_struct *curr, struct held_lock *this,
+ enum lock_usage_bit new_bit)
{
- const char *name = state_name(new_bit);
- const char *rname = state_rname(new_bit);
-
int excl_bit = exclusive_bit(new_bit);
+ int read = new_bit & 1;
+ int dir = new_bit & 2;
- if (!valid_state(curr, this, new_bit, excl_bit))
- return 0;
- if (!valid_state(curr, this, new_bit, excl_bit + 1))
- return 0;
- /*
- * just marked it hardirq-safe, check that this lock
- * took no hardirq-unsafe lock in the past:
- */
- if (!check_usage_forwards(curr, this, excl_bit, name))
- return 0;
-#if STRICT_READ_CHECKS
/*
- * just marked it hardirq-safe, check that this lock
- * took no hardirq-unsafe-read lock in the past:
+ * mark USED_IN has to look forwards -- to ensure no dependency
+ * has ENABLED state, which would allow recursion deadlocks.
+ *
+ * mark ENABLED has to look backwards -- to ensure no dependee
+ * has USED_IN state, which, again, would allow recursion deadlocks.
*/
- if (!check_usage_forwards(curr, this, excl_bit + 1, rname))
- return 0;
-#endif
- if (verbose(hlock_class(this)))
- return 2;
+ check_usage_f usage = dir ?
+ check_usage_backwards : check_usage_forwards;
- return 1;
-}
-
-static int
-mark_lock_irq_used_in_read(struct task_struct *curr, struct held_lock *this,
- int new_bit,
- int (*verbose)(struct lock_class *class))
-{
- const char *name = state_name(new_bit);
- const char *rname = state_rname(new_bit);
-
- int excl_bit = exclusive_bit(new_bit);
-
- if (!valid_state(curr, this, new_bit, excl_bit))
- return 0;
/*
- * just marked it hardirq-read-safe, check that this lock
- * took no hardirq-unsafe lock in the past:
+ * Validate that this particular lock does not have conflicting
+ * usage states.
*/
- if (!check_usage_forwards(curr, this, excl_bit, name))
- return 0;
- if (verbose(hlock_class(this)))
- return 2;
-
- return 1;
-}
-
-static int
-mark_lock_irq_enabled(struct task_struct *curr, struct held_lock *this,
- int new_bit,
- int (*verbose)(struct lock_class *class))
-{
- const char *name = state_name(new_bit);
- const char *rname = state_rname(new_bit);
-
- int excl_bit = exclusive_bit(new_bit);
-
if (!valid_state(curr, this, new_bit, excl_bit))
return 0;
- if (!valid_state(curr, this, new_bit, excl_bit + 1))
- return 0;
+
/*
- * just marked it hardirq-unsafe, check that no hardirq-safe
- * lock in the system ever took it in the past:
+ * Validate that the lock dependencies don't have conflicting usage
+ * states.
*/
- if (!check_usage_backwards(curr, this, excl_bit, name))
+ if ((!read || !dir || STRICT_READ_CHECKS) &&
+ !usage(curr, this, excl_bit, state_name(new_bit & ~1)))
return 0;
-#if STRICT_READ_CHECKS
+
/*
- * just marked it hardirq-unsafe, check that no
- * hardirq-safe-read lock in the system ever took
- * it in the past:
+ * Check for read in write conflicts
*/
- if (!check_usage_backwards(curr, this, excl_bit + 1, rname))
- return 0;
-#endif
- if (verbose(hlock_class(this)))
- return 2;
-
- return 1;
-}
-
-static int
-mark_lock_irq_enabled_read(struct task_struct *curr, struct held_lock *this,
- int new_bit,
- int (*verbose)(struct lock_class *class))
-{
- const char *name = state_name(new_bit);
- const char *rname = state_rname(new_bit);
+ if (!read) {
+ if (!valid_state(curr, this, new_bit, excl_bit + 1))
+ return 0;
- int excl_bit = exclusive_bit(new_bit);
+ if (STRICT_READ_CHECKS &&
+ !usage(curr, this, excl_bit + 1,
+ state_name(new_bit + 1)))
+ return 0;
+ }
- if (!valid_state(curr, this, new_bit, excl_bit))
- return 0;
-#if STRICT_READ_CHECKS
- /*
- * just marked it hardirq-read-unsafe, check that no
- * hardirq-safe lock in the system ever took it in the past:
- */
- if (!check_usage_backwards(curr, this, excl_bit, name))
- return 0;
-#endif
- if (verbose(hlock_class(this)))
+ if (state_verbose(new_bit, hlock_class(this)))
return 2;
return 1;
}
-static int mark_lock_irq(struct task_struct *curr, struct held_lock *this,
- enum lock_usage_bit new_bit)
-{
- int ret = 1;
-
- switch(new_bit) {
- case LOCK_USED_IN_HARDIRQ:
- return mark_lock_irq_used_in(curr, this, new_bit,
- hardirq_verbose);
- case LOCK_USED_IN_SOFTIRQ:
- return mark_lock_irq_used_in(curr, this, new_bit,
- softirq_verbose);
- case LOCK_USED_IN_RECLAIM_FS:
- return mark_lock_irq_used_in(curr, this, new_bit,
- reclaim_verbose);
-
- case LOCK_USED_IN_HARDIRQ_READ:
- return mark_lock_irq_used_in_read(curr, this, new_bit,
- hardirq_verbose);
- case LOCK_USED_IN_SOFTIRQ_READ:
- return mark_lock_irq_used_in_read(curr, this, new_bit,
- softirq_verbose);
- case LOCK_USED_IN_RECLAIM_FS_READ:
- return mark_lock_irq_used_in_read(curr, this, new_bit,
- reclaim_verbose);
-
- case LOCK_ENABLED_HARDIRQ:
- return mark_lock_irq_enabled(curr, this, new_bit,
- hardirq_verbose);
- case LOCK_ENABLED_SOFTIRQ:
- return mark_lock_irq_enabled(curr, this, new_bit,
- softirq_verbose);
- case LOCK_ENABLED_RECLAIM_FS:
- return mark_lock_irq_enabled(curr, this, new_bit,
- reclaim_verbose);
-
- case LOCK_ENABLED_HARDIRQ_READ:
- return mark_lock_irq_enabled_read(curr, this, new_bit,
- hardirq_verbose);
- case LOCK_ENABLED_SOFTIRQ_READ:
- return mark_lock_irq_enabled_read(curr, this, new_bit,
- softirq_verbose);
- case LOCK_ENABLED_RECLAIM_FS_READ:
- return mark_lock_irq_enabled_read(curr, this, new_bit,
- reclaim_verbose);
-
- default:
- WARN_ON(1);
- break;
- }
-
- return ret;
-}
-
enum mark_type {
#define LOCKDEP_STATE(__STATE) __STATE,
#include "lockdep_states.h"
#undef LOCKDEP_STATE
};
-#define MARK_HELD_CASE(__STATE) \
- case __STATE: \
- if (hlock->read) \
- usage_bit = LOCK_ENABLED_##__STATE##_READ; \
- else \
- usage_bit = LOCK_ENABLED_##__STATE; \
- break;
-
/*
* Mark all held locks with a usage bit:
*/
for (i = 0; i < curr->lockdep_depth; i++) {
hlock = curr->held_locks + i;
- switch (mark) {
-#define LOCKDEP_STATE(__STATE) MARK_HELD_CASE(__STATE)
-#include "lockdep_states.h"
-#undef LOCKDEP_STATE
- default:
- BUG();
- }
+ usage_bit = 2 + (mark << 2); /* ENABLED */
+ if (hlock->read)
+ usage_bit += 1; /* READ */
+
+ BUG_ON(usage_bit >= LOCK_USAGE_STATES);
if (!mark_lock(curr, hlock, usage_bit))
return 0;
debug_atomic_inc(&redundant_softirqs_off);
}
-void lockdep_trace_alloc(gfp_t gfp_mask)
+static void __lockdep_trace_alloc(gfp_t gfp_mask, unsigned long flags)
{
struct task_struct *curr = current;
if (!(gfp_mask & __GFP_FS))
return;
- if (DEBUG_LOCKS_WARN_ON(irqs_disabled()))
+ if (DEBUG_LOCKS_WARN_ON(irqs_disabled_flags(flags)))
return;
mark_held_locks(curr, RECLAIM_FS);
}
+static void check_flags(unsigned long flags);
+
+void lockdep_trace_alloc(gfp_t gfp_mask)
+{
+ unsigned long flags;
+
+ if (unlikely(current->lockdep_recursion))
+ return;
+
+ raw_local_irq_save(flags);
+ check_flags(flags);
+ current->lockdep_recursion = 1;
+ __lockdep_trace_alloc(gfp_mask, flags);
+ current->lockdep_recursion = 0;
+ raw_local_irq_restore(flags);
+}
+
static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)
{
/*
return 0;
}
+void lockdep_trace_alloc(gfp_t gfp_mask)
+{
+}
+
#endif
/*
void lockdep_init_map(struct lockdep_map *lock, const char *name,
struct lock_class_key *key, int subclass)
{
- if (unlikely(!debug_locks))
+ lock->class_cache = NULL;
+#ifdef CONFIG_LOCK_STAT
+ lock->cpu = raw_smp_processor_id();
+#endif
+
+ if (DEBUG_LOCKS_WARN_ON(!name)) {
+ lock->name = "NULL";
return;
+ }
+
+ lock->name = name;
if (DEBUG_LOCKS_WARN_ON(!key))
return;
- if (DEBUG_LOCKS_WARN_ON(!name))
- return;
/*
* Sanity check, the lock-class key must be persistent:
*/
DEBUG_LOCKS_WARN_ON(1);
return;
}
- lock->name = name;
lock->key = key;
- lock->class_cache = NULL;
-#ifdef CONFIG_LOCK_STAT
- lock->cpu = raw_smp_processor_id();
-#endif
+
+ if (unlikely(!debug_locks))
+ return;
+
if (subclass)
register_lock_class(lock, subclass, 1);
}
debug_locks_off();
printk("BUG: MAX_LOCKDEP_SUBCLASSES too low!\n");
printk("turning off the locking correctness validator.\n");
+ dump_stack();
return 0;
}
debug_locks_off();
printk("BUG: MAX_LOCK_DEPTH too low!\n");
printk("turning off the locking correctness validator.\n");
+ dump_stack();
return 0;
}
{
unsigned long flags;
+ trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
+
if (unlikely(current->lockdep_recursion))
return;
{
unsigned long flags;
+ trace_lock_release(lock, nested, ip);
+
if (unlikely(current->lockdep_recursion))
return;
hlock->holdtime_stamp = now;
}
+ trace_lock_acquired(lock, ip, waittime);
+
stats = get_lock_stats(hlock_class(hlock));
if (waittime) {
if (hlock->read)
{
unsigned long flags;
+ trace_lock_contended(lock, ip);
+
if (unlikely(!lock_stat))
return;
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff