rcutorture: Emulate userspace sojourn during call_rcu() floods

During an actual call_rcu() flood, there would be frequent trips to
userspace (in-kernel call_rcu() floods must be otherwise housebroken).
Userspace execution allows a great many things to interrupt execution,
and rcutorture needs to also allow such interruptions.  This commit
therefore causes call_rcu() floods to occasionally invoke schedule(),
thus preventing spurious rcutorture failures due to other parts of the
kernel becoming irate at the call_rcu() flood events.

Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index fce4e7e6f50247be58d14d258efdbbfbcff8f53f..c44e5307afcce4b2f24393b0feca7c0fe9732c81 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -1713,12 +1713,14 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
 }
 
 // Give the scheduler a chance, even on nohz_full CPUs.
-static void rcu_torture_fwd_prog_cond_resched(void)
+static void rcu_torture_fwd_prog_cond_resched(unsigned long iter)
 {
        if (IS_ENABLED(CONFIG_PREEMPT) && IS_ENABLED(CONFIG_NO_HZ_FULL)) {
-               if (need_resched())
+               // Real call_rcu() floods hit userspace, so emulate that.
+               if (need_resched() || (iter & 0xfff))
                        schedule();
        } else {
+               // No userspace emulation: CB invocation throttles call_rcu()
                cond_resched();
        }
 }
@@ -1746,7 +1748,7 @@ static unsigned long rcu_torture_fwd_prog_cbfree(void)
                spin_unlock_irqrestore(&rcu_fwd_lock, flags);
                kfree(rfcp);
                freed++;
-               rcu_torture_fwd_prog_cond_resched();
+               rcu_torture_fwd_prog_cond_resched(freed);
        }
        return freed;
 }
@@ -1790,7 +1792,7 @@ static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries)
                udelay(10);
                cur_ops->readunlock(idx);
                if (!fwd_progress_need_resched || need_resched())
-                       rcu_torture_fwd_prog_cond_resched();
+                       rcu_torture_fwd_prog_cond_resched(1);
        }
        (*tested_tries)++;
        if (!time_before(jiffies, stopat) &&
@@ -1875,7 +1877,7 @@ static void rcu_torture_fwd_prog_cr(void)
                        rfcp->rfc_gps = 0;
                }
                cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
-               rcu_torture_fwd_prog_cond_resched();
+               rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);
        }
        stoppedat = jiffies;
        n_launders_cb_snap = READ_ONCE(n_launders_cb);