cpufreq: Reduce cpufreq_update_util() overhead a bit

Use the observation that cpufreq_update_util() is only called
by the scheduler with rq->lock held, so the callers of
cpufreq_set_update_util_data() can use synchronize_sched()
instead of synchronize_rcu() to wait for cpufreq_update_util()
to complete.  Moreover, if they are updated to do that,
rcu_read_(un)lock() calls in cpufreq_update_util() might be
replaced with rcu_read_(un)lock_sched(), respectively, but
those aren't really necessary, because the scheduler calls
that function from RCU-sched read-side critical sections
already.

In addition to that, if cpufreq_set_update_util_data() checks
the func field in the struct update_util_data before setting
the per-CPU pointer to it, the data->func check may be dropped
from cpufreq_update_util() as well.

Make the above changes to reduce the overhead from
cpufreq_update_util() in the scheduler paths invoking it
and to make the cleanup after removing its callbacks less
heavy-weight somewhat.

Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index dd568aaf27280fb50e933cd7c4ae1ede8aca6519..6eca12ab71d7a5efa3b89dc2032e647ec0b3bc84 100644 (file)
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -115,12 +115,15 @@ static DEFINE_PER_CPU(struct update_util_data *, cpufreq_update_util_data);
  * to call from cpufreq_update_util().  That function will be called from an RCU
  * read-side critical section, so it must not sleep.
  *
- * Callers must use RCU callbacks to free any memory that might be accessed
- * via the old update_util_data pointer or invoke synchronize_rcu() right after
- * this function to avoid use-after-free.
+ * Callers must use RCU-sched callbacks to free any memory that might be
+ * accessed via the old update_util_data pointer or invoke synchronize_sched()
+ * right after this function to avoid use-after-free.
  */
 void cpufreq_set_update_util_data(int cpu, struct update_util_data *data)
 {
+       if (WARN_ON(data && !data->func))
+               return;
+
        rcu_assign_pointer(per_cpu(cpufreq_update_util_data, cpu), data);
 }
 EXPORT_SYMBOL_GPL(cpufreq_set_update_util_data);
@@ -133,18 +136,24 @@ EXPORT_SYMBOL_GPL(cpufreq_set_update_util_data);
  *
  * This function is called by the scheduler on every invocation of
  * update_load_avg() on the CPU whose utilization is being updated.
+ *
+ * It can only be called from RCU-sched read-side critical sections.
  */
 void cpufreq_update_util(u64 time, unsigned long util, unsigned long max)
 {
        struct update_util_data *data;
 
-       rcu_read_lock();
+#ifdef CONFIG_LOCKDEP
+       WARN_ON(debug_locks && !rcu_read_lock_sched_held());
+#endif
 
-       data = rcu_dereference(*this_cpu_ptr(&cpufreq_update_util_data));
-       if (data && data->func)
+       data = rcu_dereference_sched(*this_cpu_ptr(&cpufreq_update_util_data));
+       /*
+        * If this isn't inside of an RCU-sched read-side critical section, data
+        * may become NULL after the check below.
+        */
+       if (data)
                data->func(data, time, util, max);
-
-       rcu_read_unlock();
 }
 
 /* Flag to suspend/resume CPUFreq governors */

diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 70079e21fa2d5bf6e981e3975685fe1ccfc430c2..db46190bb246a1479f860445eb474fabfb93f02e 100644 (file)
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -280,7 +280,7 @@ static inline void gov_clear_update_util(struct cpufreq_policy *policy)
        for_each_cpu(i, policy->cpus)
                cpufreq_set_update_util_data(i, NULL);
 
-       synchronize_rcu();
+       synchronize_sched();
 }
 
 static void gov_cancel_work(struct cpufreq_policy *policy)

diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index f4d85c2ae7b18de7d986504e3eb089944f90c431..2165d2b2fc35925e8c12631ac0085b39b2f123fc 100644 (file)
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1168,7 +1168,7 @@ static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
        pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);
 
        cpufreq_set_update_util_data(cpu_num, NULL);
-       synchronize_rcu();
+       synchronize_sched();
 
        if (hwp_active)
                return;
@@ -1426,7 +1426,7 @@ out:
        for_each_online_cpu(cpu) {
                if (all_cpu_data[cpu]) {
                        cpufreq_set_update_util_data(cpu, NULL);
-                       synchronize_rcu();
+                       synchronize_sched();
                        kfree(all_cpu_data[cpu]);
                }
        }