sched, block: Unify cache detection

The block layer has some code trying to determine if two CPUs share a
cache, the scheduler has a similar function. Expose the function used
by the scheduler and make the block layer use it, thereby removing the
block layers usage of CONFIG_SCHED* and topology bits.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Jens Axboe <axboe@kernel.dk>
Link: http://lkml.kernel.org/r/1327579450.2446.95.camel@twins

diff --git a/block/blk-softirq.c b/block/blk-softirq.c
index 1366a89d8e663c5cf968557c9fdab6bc462fef39..467c8de8864200cad58d66a283183d5ed5868f70 100644 (file)
--- a/block/blk-softirq.c
+++ b/block/blk-softirq.c
@@ -8,6 +8,7 @@
 #include <linux/blkdev.h>
 #include <linux/interrupt.h>
 #include <linux/cpu.h>
+#include <linux/sched.h>
 
 #include "blk.h"
 
@@ -103,9 +104,10 @@ static struct notifier_block __cpuinitdata blk_cpu_notifier = {
 
 void __blk_complete_request(struct request *req)
 {
-       int ccpu, cpu, group_cpu = NR_CPUS;
+       int ccpu, cpu;
        struct request_queue *q = req->q;
        unsigned long flags;
+       bool shared = false;
 
        BUG_ON(!q->softirq_done_fn);
 
@@ -117,22 +119,20 @@ void __blk_complete_request(struct request *req)
         */
        if (req->cpu != -1) {
                ccpu = req->cpu;
-               if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) {
-                       ccpu = blk_cpu_to_group(ccpu);
-                       group_cpu = blk_cpu_to_group(cpu);
-               }
+               if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
+                       shared = cpus_share_cache(cpu, ccpu);
        } else
                ccpu = cpu;
 
        /*
-        * If current CPU and requested CPU are in the same group, running
-        * softirq in current CPU. One might concern this is just like
+        * If current CPU and requested CPU share a cache, run the softirq on
+        * the current CPU. One might concern this is just like
         * QUEUE_FLAG_SAME_FORCE, but actually not. blk_complete_request() is
         * running in interrupt handler, and currently I/O controller doesn't
         * support multiple interrupts, so current CPU is unique actually. This
         * avoids IPI sending from current CPU to the first CPU of a group.
         */
-       if (ccpu == cpu || ccpu == group_cpu) {
+       if (ccpu == cpu || shared) {
                struct list_head *list;
 do_local:
                list = &__get_cpu_var(blk_cpu_done);

diff --git a/block/blk.h b/block/blk.h
index 7efd772336de998be8c53b01a1805ae63069de62..df5b59acbdff081eb2ca1480470f90b5e2fe5546 100644 (file)
--- a/block/blk.h
+++ b/block/blk.h
@@ -164,22 +164,6 @@ static inline int queue_congestion_off_threshold(struct request_queue *q)
        return q->nr_congestion_off;
 }
 
-static inline int blk_cpu_to_group(int cpu)
-{
-       int group = NR_CPUS;
-#ifdef CONFIG_SCHED_MC
-       const struct cpumask *mask = cpu_coregroup_mask(cpu);
-       group = cpumask_first(mask);
-#elif defined(CONFIG_SCHED_SMT)
-       group = cpumask_first(topology_thread_cpumask(cpu));
-#else
-       return cpu;
-#endif
-       if (likely(group < NR_CPUS))
-               return group;
-       return cpu;
-}
-
 /*
  * Contribute to IO statistics IFF:
  *

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 513f524598724d14eaf62384e40712fc1624006c..0e1959568836f2b6328da27b2f492e90f0742c08 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1052,6 +1052,8 @@ static inline int test_sd_parent(struct sched_domain *sd, int flag)
 unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu);
 unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu);
 
+bool cpus_share_cache(int this_cpu, int that_cpu);
+
 #else /* CONFIG_SMP */
 
 struct sched_domain_attr;
@@ -1061,6 +1063,12 @@ partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
                        struct sched_domain_attr *dattr_new)
 {
 }
+
+static inline bool cpus_share_cache(int this_cpu, int that_cpu)
+{
+       return true;
+}
+
 #endif /* !CONFIG_SMP */
 
 

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5255c9d2e053225173dfea134e7e243ad0e80891..d7c43227311d37c4182c0ed196c3fb84fa38a491 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1507,7 +1507,7 @@ static int ttwu_activate_remote(struct task_struct *p, int wake_flags)
 }
 #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
 
-static inline int ttwu_share_cache(int this_cpu, int that_cpu)
+bool cpus_share_cache(int this_cpu, int that_cpu)
 {
        return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
 }
@@ -1518,7 +1518,7 @@ static void ttwu_queue(struct task_struct *p, int cpu)
        struct rq *rq = cpu_rq(cpu);
 
 #if defined(CONFIG_SMP)
-       if (sched_feat(TTWU_QUEUE) && !ttwu_share_cache(smp_processor_id(), cpu)) {
+       if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
                sched_clock_cpu(cpu); /* sync clocks x-cpu */
                ttwu_queue_remote(p, cpu);
                return;
@@ -5754,7 +5754,7 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
  *
  * Also keep a unique ID per domain (we use the first cpu number in
  * the cpumask of the domain), this allows us to quickly tell if
- * two cpus are in the same cache domain, see ttwu_share_cache().
+ * two cpus are in the same cache domain, see cpus_share_cache().
  */
 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
 DEFINE_PER_CPU(int, sd_llc_id);