summaryrefslogtreecommitdiff
path: root/kernel/rcu/tree_plugin.h
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2015-07-31 16:04:45 -0700
committerPaul E. McKenney <paulmck@linux.vnet.ibm.com>2015-09-20 21:16:18 -0700
commitb9585e940a0d78770cda8f9aebf81b17b4d19e6d (patch)
tree393ed9b496b85179ebf4112fc9d11d8b4f1de5a2 /kernel/rcu/tree_plugin.h
parent7922cd0e562cb2b8da2c8a0afda2e1c9bb4a94e2 (diff)
downloadlinux-sh-b9585e940a0d78770cda8f9aebf81b17b4d19e6d.tar.gz
rcu: Consolidate tree setup for synchronize_rcu_expedited()
This commit replaces sync_rcu_preempt_exp_init1(() and sync_rcu_preempt_exp_init2() with sync_exp_reset_tree_hotplug() and sync_exp_reset_tree(), which will also be used by synchronize_sched_expedited(), and sync_rcu_exp_select_nodes(), which contains code specific to synchronize_rcu_expedited(). Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Diffstat (limited to 'kernel/rcu/tree_plugin.h')
-rw-r--r--kernel/rcu/tree_plugin.h102
1 files changed, 18 insertions, 84 deletions
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index e73be8539978..62d05413b7ba 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -536,86 +536,28 @@ void synchronize_rcu(void)
EXPORT_SYMBOL_GPL(synchronize_rcu);
/*
- * Snapshot the tasks blocking the newly started preemptible-RCU expedited
- * grace period for the specified rcu_node structure, phase 1. If there
- * are such tasks, set the ->expmask bits up the rcu_node tree and also
- * set the ->expmask bits on the leaf rcu_node structures to tell phase 2
- * that work is needed here.
- *
- * Caller must hold the root rcu_node's exp_funnel_mutex.
+ * Select the nodes that the upcoming expedited grace period needs
+ * to wait for.
*/
-static void
-sync_rcu_preempt_exp_init1(struct rcu_state *rsp, struct rcu_node *rnp)
+static void sync_rcu_exp_select_nodes(struct rcu_state *rsp)
{
unsigned long flags;
- unsigned long mask;
- struct rcu_node *rnp_up;
+ struct rcu_node *rnp;
- raw_spin_lock_irqsave(&rnp->lock, flags);
- smp_mb__after_unlock_lock();
- WARN_ON_ONCE(rnp->expmask);
- WARN_ON_ONCE(rnp->exp_tasks);
- if (!rcu_preempt_has_tasks(rnp)) {
- /* No blocked tasks, nothing to do. */
- raw_spin_unlock_irqrestore(&rnp->lock, flags);
- return;
- }
- /* Call for Phase 2 and propagate ->expmask bits up the tree. */
- rnp->expmask = 1;
- rnp_up = rnp;
- while (rnp_up->parent) {
- mask = rnp_up->grpmask;
- rnp_up = rnp_up->parent;
- if (rnp_up->expmask & mask)
- break;
- raw_spin_lock(&rnp_up->lock); /* irqs already off */
+ sync_exp_reset_tree(rsp);
+ rcu_for_each_leaf_node(rsp, rnp) {
+ raw_spin_lock_irqsave(&rnp->lock, flags);
smp_mb__after_unlock_lock();
- rnp_up->expmask |= mask;
- raw_spin_unlock(&rnp_up->lock); /* irqs still off */
- }
- raw_spin_unlock_irqrestore(&rnp->lock, flags);
-}
-
-/*
- * Snapshot the tasks blocking the newly started preemptible-RCU expedited
- * grace period for the specified rcu_node structure, phase 2. If the
- * leaf rcu_node structure has its ->expmask field set, check for tasks.
- * If there are some, clear ->expmask and set ->exp_tasks accordingly,
- * then initiate RCU priority boosting. Otherwise, clear ->expmask and
- * invoke rcu_report_exp_rnp() to clear out the upper-level ->expmask bits,
- * enabling rcu_read_unlock_special() to do the bit-clearing.
- *
- * Caller must hold the root rcu_node's exp_funnel_mutex.
- */
-static void
-sync_rcu_preempt_exp_init2(struct rcu_state *rsp, struct rcu_node *rnp)
-{
- unsigned long flags;
-
- raw_spin_lock_irqsave(&rnp->lock, flags);
- smp_mb__after_unlock_lock();
- if (!rnp->expmask) {
- /* Phase 1 didn't do anything, so Phase 2 doesn't either. */
- raw_spin_unlock_irqrestore(&rnp->lock, flags);
- return;
- }
-
- /* Phase 1 is over. */
- rnp->expmask = 0;
-
- /*
- * If there are still blocked tasks, set up ->exp_tasks so that
- * rcu_read_unlock_special() will wake us and then boost them.
- */
- if (rcu_preempt_has_tasks(rnp)) {
- rnp->exp_tasks = rnp->blkd_tasks.next;
- rcu_initiate_boost(rnp, flags); /* releases rnp->lock */
- return;
+ rnp->expmask = 0; /* No per-CPU component yet. */
+ if (!rcu_preempt_has_tasks(rnp)) {
+ /* FIXME: Want __rcu_report_exp_rnp() here. */
+ raw_spin_unlock_irqrestore(&rnp->lock, flags);
+ } else {
+ rnp->exp_tasks = rnp->blkd_tasks.next;
+ rcu_initiate_boost(rnp, flags);
+ }
+ rcu_report_exp_rnp(rsp, rnp, false);
}
-
- /* No longer any blocked tasks, so undo bit setting. */
- raw_spin_unlock_irqrestore(&rnp->lock, flags);
- rcu_report_exp_rnp(rsp, rnp, false);
}
/**
@@ -648,16 +590,8 @@ void synchronize_rcu_expedited(void)
/* force all RCU readers onto ->blkd_tasks lists. */
synchronize_sched_expedited();
- /*
- * Snapshot current state of ->blkd_tasks lists into ->expmask.
- * Phase 1 sets bits and phase 2 permits rcu_read_unlock_special()
- * to start clearing them. Doing this in one phase leads to
- * strange races between setting and clearing bits, so just say "no"!
- */
- rcu_for_each_leaf_node(rsp, rnp)
- sync_rcu_preempt_exp_init1(rsp, rnp);
- rcu_for_each_leaf_node(rsp, rnp)
- sync_rcu_preempt_exp_init2(rsp, rnp);
+ /* Initialize the rcu_node tree in preparation for the wait. */
+ sync_rcu_exp_select_nodes(rsp);
/* Wait for snapshotted ->blkd_tasks lists to drain. */
rnp = rcu_get_root(rsp);