#include <linux/cpu.h>
#include <linux/mutex.h>
#include <linux/module.h>
-#include <linux/kernel_stat.h>
+#include <linux/hardirq.h>
#ifdef CONFIG_DEBUG_LOCK_ALLOC
static struct lock_class_key rcu_lock_key;
struct lockdep_map rcu_lock_map =
STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
EXPORT_SYMBOL_GPL(rcu_lock_map);
-#endif
-
-int rcu_scheduler_active __read_mostly;
-/*
- * Awaken the corresponding synchronize_rcu() instance now that a
- * grace period has elapsed.
- */
-void wakeme_after_rcu(struct rcu_head *head)
-{
- struct rcu_synchronize *rcu;
+static struct lock_class_key rcu_bh_lock_key;
+struct lockdep_map rcu_bh_lock_map =
+ STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_bh", &rcu_bh_lock_key);
+EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
- rcu = container_of(head, struct rcu_synchronize, head);
- complete(&rcu->completion);
-}
-
-#ifndef CONFIG_TINY_RCU
+static struct lock_class_key rcu_sched_lock_key;
+struct lockdep_map rcu_sched_lock_map =
+ STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_sched", &rcu_sched_lock_key);
+EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
+#endif
-#ifdef CONFIG_TREE_PREEMPT_RCU
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
-/**
- * synchronize_rcu - wait until a grace period has elapsed.
- *
- * Control will return to the caller some time after a full grace
- * period has elapsed, in other words after all currently executing RCU
- * read-side critical sections have completed. RCU read-side critical
- * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
- * and may be nested.
- */
-void synchronize_rcu(void)
+int debug_lockdep_rcu_enabled(void)
{
- struct rcu_synchronize rcu;
-
- if (!rcu_scheduler_active)
- return;
-
- init_completion(&rcu.completion);
- /* Will wake me after RCU finished. */
- call_rcu(&rcu.head, wakeme_after_rcu);
- /* Wait for it. */
- wait_for_completion(&rcu.completion);
+ return rcu_scheduler_active && debug_locks &&
+ current->lockdep_recursion == 0;
}
-EXPORT_SYMBOL_GPL(synchronize_rcu);
-
-#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
+EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
/**
- * synchronize_sched - wait until an rcu-sched grace period has elapsed.
- *
- * Control will return to the caller some time after a full rcu-sched
- * grace period has elapsed, in other words after all currently executing
- * rcu-sched read-side critical sections have completed. These read-side
- * critical sections are delimited by rcu_read_lock_sched() and
- * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(),
- * local_irq_disable(), and so on may be used in place of
- * rcu_read_lock_sched().
- *
- * This means that all preempt_disable code sequences, including NMI and
- * hardware-interrupt handlers, in progress on entry will have completed
- * before this primitive returns. However, this does not guarantee that
- * softirq handlers will have completed, since in some kernels, these
- * handlers can run in process context, and can block.
+ * rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section?
*
- * This primitive provides the guarantees made by the (now removed)
- * synchronize_kernel() API. In contrast, synchronize_rcu() only
- * guarantees that rcu_read_lock() sections will have completed.
- * In "classic RCU", these two guarantees happen to be one and
- * the same, but can differ in realtime RCU implementations.
- */
-void synchronize_sched(void)
-{
- struct rcu_synchronize rcu;
-
- if (rcu_blocking_is_gp())
- return;
-
- init_completion(&rcu.completion);
- /* Will wake me after RCU finished. */
- call_rcu_sched(&rcu.head, wakeme_after_rcu);
- /* Wait for it. */
- wait_for_completion(&rcu.completion);
-}
-EXPORT_SYMBOL_GPL(synchronize_sched);
-
-/**
- * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
+ * Check for bottom half being disabled, which covers both the
+ * CONFIG_PROVE_RCU and not cases. Note that if someone uses
+ * rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled)
+ * will show the situation.
*
- * Control will return to the caller some time after a full rcu_bh grace
- * period has elapsed, in other words after all currently executing rcu_bh
- * read-side critical sections have completed. RCU read-side critical
- * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
- * and may be nested.
+ * Check debug_lockdep_rcu_enabled() to prevent false positives during boot.
*/
-void synchronize_rcu_bh(void)
+int rcu_read_lock_bh_held(void)
{
- struct rcu_synchronize rcu;
-
- if (rcu_blocking_is_gp())
- return;
-
- init_completion(&rcu.completion);
- /* Will wake me after RCU finished. */
- call_rcu_bh(&rcu.head, wakeme_after_rcu);
- /* Wait for it. */
- wait_for_completion(&rcu.completion);
+ if (!debug_lockdep_rcu_enabled())
+ return 1;
+ return in_softirq();
}
-EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
+EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
-#endif /* #ifndef CONFIG_TINY_RCU */
-
-static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
- unsigned long action, void *hcpu)
-{
- return rcu_cpu_notify(self, action, hcpu);
-}
+#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
-void __init rcu_init(void)
+/*
+ * Awaken the corresponding synchronize_rcu() instance now that a
+ * grace period has elapsed.
+ */
+void wakeme_after_rcu(struct rcu_head *head)
{
- int i;
-
- __rcu_init();
- cpu_notifier(rcu_barrier_cpu_hotplug, 0);
+ struct rcu_synchronize *rcu;
- /*
- * We don't need protection against CPU-hotplug here because
- * this is called early in boot, before either interrupts
- * or the scheduler are operational.
- */
- for_each_online_cpu(i)
- rcu_barrier_cpu_hotplug(NULL, CPU_UP_PREPARE, (void *)(long)i);
+ rcu = container_of(head, struct rcu_synchronize, head);
+ complete(&rcu->completion);
}
-void rcu_scheduler_starting(void)
+#ifdef CONFIG_PROVE_RCU
+/*
+ * wrapper function to avoid #include problems.
+ */
+int rcu_my_thread_group_empty(void)
{
- WARN_ON(num_online_cpus() != 1);
- WARN_ON(nr_context_switches() > 0);
- rcu_scheduler_active = 1;
+ return thread_group_empty(current);
}
+EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty);
+#endif /* #ifdef CONFIG_PROVE_RCU */
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff