rcu: Further cleanups of use of lastcomp

Now that a copy of the rsp->completed flag is available in all
rcu_node structures, make full use of it.  It is still
legitimate to access rsp->completed while holding the root
rcu_node structure's lock, however.

Also, tighten up force_quiescent_state()'s checks for end of
current grace period.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1258170699933-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 3df04381ea3ecb0c6d794ee301767bd905f6e0cc..24bbf2ce06056f512c85542355f63e4a37e9f1dc 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -817,7 +817,7 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
 
        rnp = rdp->mynode;
        spin_lock_irqsave(&rnp->lock, flags);
-       if (lastcomp != ACCESS_ONCE(rsp->completed)) {
+       if (lastcomp != rnp->completed) {
 
                /*
                 * Someone beat us to it for this grace period, so leave.
@@ -935,7 +935,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
 static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
 {
        unsigned long flags;
-       long lastcomp;
        unsigned long mask;
        struct rcu_data *rdp = rsp->rda[cpu];
        struct rcu_node *rnp;
@@ -971,7 +970,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
                spin_unlock(&rnp->lock);        /* irqs remain disabled. */
                rnp = rnp->parent;
        } while (rnp != NULL);
-       lastcomp = rsp->completed;
 
        spin_unlock_irqrestore(&rsp->onofflock, flags);
 
@@ -1145,7 +1143,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
        rcu_for_each_leaf_node(rsp, rnp) {
                mask = 0;
                spin_lock_irqsave(&rnp->lock, flags);
-               if (rsp->completed != lastcomp) {
+               if (rnp->completed != lastcomp) {
                        spin_unlock_irqrestore(&rnp->lock, flags);
                        return 1;
                }
@@ -1159,7 +1157,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
                        if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))
                                mask |= bit;
                }
-               if (mask != 0 && rsp->completed == lastcomp) {
+               if (mask != 0 && rnp->completed == lastcomp) {
 
                        /* cpu_quiet_msk() releases rnp->lock. */
                        cpu_quiet_msk(mask, rsp, rnp, flags);
@@ -1196,7 +1194,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
        lastcomp = rsp->gpnum - 1;
        signaled = rsp->signaled;
        rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
-       if (lastcomp == rsp->gpnum) {
+       if(!rcu_gp_in_progress(rsp)) {
                rsp->n_force_qs_ngp++;
                spin_unlock(&rnp->lock);
                goto unlock_ret;  /* no GP in progress, time updated. */
@@ -1224,7 +1222,8 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
                /* Update state, record completion counter. */
                forcenow = 0;
                spin_lock(&rnp->lock);
-               if (lastcomp == rsp->completed &&
+               if (lastcomp + 1 == rsp->gpnum &&
+                   lastcomp == rsp->completed &&
                    rsp->signaled == signaled) {
                        rsp->signaled = RCU_FORCE_QS;
                        rsp->completed_fqs = lastcomp;