rcu: performance fixes to TINY_PREEMPT_RCU callback checking

This commit tightens up checks in rcu_preempt_check_callbacks() to avoid
unnecessary special handling at rcu_read_unlock() time.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index c5bea1137dcb87ecd9411b3d7eb308d8488540cf..6ceca4f745ffa1f4535c69467ea59704e2ddbe97 100644 (file)
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -72,7 +72,7 @@ static void rcu_report_exp_done(void);
 /*
  * Return true if the CPU has not yet responded to the current grace period.
  */
-static int rcu_cpu_cur_gp(void)
+static int rcu_cpu_blocking_cur_gp(void)
 {
        return rcu_preempt_ctrlblk.gpcpu != rcu_preempt_ctrlblk.gpnum;
 }
@@ -229,7 +229,7 @@ void rcu_preempt_note_context_switch(void)
                 * cannot end.
                 */
                list_add(&t->rcu_node_entry, &rcu_preempt_ctrlblk.blkd_tasks);
-               if (rcu_cpu_cur_gp())
+               if (rcu_cpu_blocking_cur_gp())
                        rcu_preempt_ctrlblk.gp_tasks = &t->rcu_node_entry;
        }
 
@@ -368,12 +368,16 @@ static void rcu_preempt_check_callbacks(void)
 {
        struct task_struct *t = current;
 
-       if (!rcu_preempt_running_reader() && rcu_preempt_gp_in_progress())
+       if (rcu_preempt_gp_in_progress() &&
+           (!rcu_preempt_running_reader() ||
+            !rcu_cpu_blocking_cur_gp()))
                rcu_preempt_cpu_qs();
        if (&rcu_preempt_ctrlblk.rcb.rcucblist !=
            rcu_preempt_ctrlblk.rcb.donetail)
                raise_softirq(RCU_SOFTIRQ);
-       if (rcu_preempt_gp_in_progress() && rcu_preempt_running_reader())
+       if (rcu_preempt_gp_in_progress() &&
+           rcu_cpu_blocking_cur_gp() &&
+           rcu_preempt_running_reader())
                t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
 }