sched: Rework CPU hotplug task selection

The CPU hotplug task selection is the only place where we used
put_prev_task() on a task that is not current. While looking at that,
it occured to me that we can simplify all that by by using a custom
pick loop.

Since we don't need to put current, we can do away with the fake task
too.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lwe@gmail.com>
Cc: Valentin Schneider <valentin.schneider@arm.com>
Cc: mingo@kernel.org
Cc: Phil Auld <pauld@redhat.com>
Cc: Julien Desfossez <jdesfossez@digitalocean.com>
Cc: Nishanth Aravamudan <naravamudan@digitalocean.com>

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9a821ff68502298885b6ee215595dabf291f624c..364b6d7da2bea2a9b4b56a616f14d80e470dbd4c 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6082,21 +6082,22 @@ static void calc_load_migrate(struct rq *rq)
                atomic_long_add(delta, &calc_load_tasks);
 }
 
-static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
+static struct task_struct *__pick_migrate_task(struct rq *rq)
 {
-}
+       const struct sched_class *class;
+       struct task_struct *next;
 
-static const struct sched_class fake_sched_class = {
-       .put_prev_task = put_prev_task_fake,
-};
+       for_each_class(class) {
+               next = class->pick_next_task(rq, NULL, NULL);
+               if (next) {
+                       next->sched_class->put_prev_task(rq, next);
+                       return next;
+               }
+       }
 
-static struct task_struct fake_task = {
-       /*
-        * Avoid pull_{rt,dl}_task()
-        */
-       .prio = MAX_PRIO + 1,
-       .sched_class = &fake_sched_class,
-};
+       /* The idle class should always have a runnable task */
+       BUG();
+}
 
 /*
  * Migrate all tasks from the rq, sleeping tasks will be migrated by
@@ -6139,12 +6140,7 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
                if (rq->nr_running == 1)
                        break;
 
-               /*
-                * pick_next_task() assumes pinned rq->lock:
-                */
-               next = pick_next_task(rq, &fake_task, rf);
-               BUG_ON(!next);
-               put_prev_task(rq, next);
+               next = __pick_migrate_task(rq);
 
                /*
                 * Rules for changing task_struct::cpus_mask are holding

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ea48aa5daeee8fb283c03c9c94f8d0d390a32471..b3449d0dd7f08e36992ccaca7aa31f7b9e679af4 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1751,6 +1751,7 @@ struct sched_class {
 
 static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
 {
+       WARN_ON_ONCE(rq->curr != prev);
        prev->sched_class->put_prev_task(rq, prev);
 }