Merge branch 'for-linus' of git://neil.brown.name/md

[safe/jmp/linux-2.6] / kernel / rcutree.h

diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index e6ab31c..1899023 100644 (file)
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -106,6 +106,18 @@ struct rcu_node {
                                /*  blocked_tasks[] array. */
 } ____cacheline_internodealigned_in_smp;
 
+/*
+ * Do a full breadth-first scan of the rcu_node structures for the
+ * specified rcu_state structure.
+ */
+#define rcu_for_each_node_breadth_first(rsp, rnp) \
+       for ((rnp) = &(rsp)->node[0]; \
+            (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++)
+
+#define rcu_for_each_leaf_node(rsp, rnp) \
+       for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \
+            (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++)
+
 /* Index values for nxttail array in struct rcu_data. */
 #define RCU_DONE_TAIL          0       /* Also RCU_WAIT head. */
 #define RCU_WAIT_TAIL          1       /* Also RCU_NEXT_READY head. */
@@ -155,6 +167,10 @@ struct rcu_data {
        struct rcu_head *nxtlist;
        struct rcu_head **nxttail[RCU_NEXT_SIZE];
        long            qlen;           /* # of queued callbacks */
+       long            qlen_last_fqs_check;
+                                       /* qlen at last check for QS forcing */
+       unsigned long   n_force_qs_snap;
+                                       /* did other CPU force QS recently? */
        long            blimit;         /* Upper limit on a processed batch */
 
 #ifdef CONFIG_NO_HZ
@@ -185,9 +201,10 @@ struct rcu_data {
 };
 
 /* Values for signaled field in struct rcu_state. */
-#define RCU_GP_INIT            0       /* Grace period being initialized. */
-#define RCU_SAVE_DYNTICK       1       /* Need to scan dyntick state. */
-#define RCU_FORCE_QS           2       /* Need to force quiescent state. */
+#define RCU_GP_IDLE            0       /* No grace period in progress. */
+#define RCU_GP_INIT            1       /* Grace period being initialized. */
+#define RCU_SAVE_DYNTICK       2       /* Need to scan dyntick state. */
+#define RCU_FORCE_QS           3       /* Need to force quiescent state. */
 #ifdef CONFIG_NO_HZ
 #define RCU_SIGNAL_INIT                RCU_SAVE_DYNTICK
 #else /* #ifdef CONFIG_NO_HZ */
@@ -232,7 +249,15 @@ struct rcu_state {
        /* End  of fields guarded by root rcu_node's lock. */
 
        spinlock_t onofflock;                   /* exclude on/offline and */
-                                               /*  starting new GP. */
+                                               /*  starting new GP.  Also */
+                                               /*  protects the following */
+                                               /*  orphan_cbs fields. */
+       struct rcu_head *orphan_cbs_list;       /* list of rcu_head structs */
+                                               /*  orphaned by all CPUs in */
+                                               /*  a given leaf rcu_node */
+                                               /*  going offline. */
+       struct rcu_head **orphan_cbs_tail;      /* And tail pointer. */
+       long orphan_qlen;                       /* Number of orphaned cbs. */
        spinlock_t fqslock;                     /* Only one task forcing */
                                                /*  quiescent states. */
        unsigned long jiffies_force_qs;         /* Time at which to invoke */
@@ -282,9 +307,9 @@ static void rcu_print_task_stall(struct rcu_node *rnp);
 #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
 #ifdef CONFIG_HOTPLUG_CPU
-static void rcu_preempt_offline_tasks(struct rcu_state *rsp,
-                                     struct rcu_node *rnp,
-                                     struct rcu_data *rdp);
+static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
+                                    struct rcu_node *rnp,
+                                    struct rcu_data *rdp);
 static void rcu_preempt_offline_cpu(int cpu);
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 static void rcu_preempt_check_callbacks(int cpu);
@@ -293,6 +318,7 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
 static int rcu_preempt_pending(int cpu);
 static int rcu_preempt_needs_cpu(int cpu);
 static void __cpuinit rcu_preempt_init_percpu_data(int cpu);
+static void rcu_preempt_send_cbs_to_orphanage(void);
 static void __init __rcu_init_preempt(void);
 
 #endif /* #else #ifdef RCU_TREE_NONCORE */