sched/fair: Combine detach into dequeue when migrating task

When we are migrating task out of the CPU, we can combine detach and
propagation into dequeue_entity() to save the detach_entity_cfs_rq()
in migrate_task_rq_fair().

This optimization is like combining DO_ATTACH in the enqueue_entity()
when migrating task to the CPU. So we don't have to traverse the CFS tree
extra time to do the detach_entity_cfs_rq() -> propagate_entity_cfs_rq(),
which wouldn't be called anymore with this patch's change.

detach_task()
  deactivate_task()
    dequeue_task_fair()
      for_each_sched_entity(se)
        dequeue_entity()
          update_load_avg() /* (1) */
            detach_entity_load_avg()

  set_task_cpu()
    migrate_task_rq_fair()
      detach_entity_cfs_rq() /* (2) */
        update_load_avg();
        detach_entity_load_avg();
        propagate_entity_cfs_rq();
          for_each_sched_entity()
            update_load_avg()

This patch save the detach_entity_cfs_rq() called in (2) by doing
the detach_entity_load_avg() for a CPU migrating task inside (1)
(the task being the first se in the loop)

Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lore.kernel.org/r/20220818124805.601-6-zhouchengming@bytedance.com

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 52de830..f52e7dc 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4003,6 +4003,7 @@ static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
 #define UPDATE_TG      0x1
 #define SKIP_AGE_LOAD  0x2
 #define DO_ATTACH      0x4
+#define DO_DETACH      0x8
 
 /* Update task and its cfs_rq load average */
 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
@@ -4032,6 +4033,13 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
                attach_entity_load_avg(cfs_rq, se);
                update_tg_load_avg(cfs_rq);
 
+       } else if (flags & DO_DETACH) {
+               /*
+                * DO_DETACH means we're here from dequeue_entity()
+                * and we are migrating task out of the CPU.
+                */
+               detach_entity_load_avg(cfs_rq, se);
+               update_tg_load_avg(cfs_rq);
        } else if (decayed) {
                cfs_rq_util_change(cfs_rq, 0);
 
@@ -4292,6 +4300,7 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
 #define UPDATE_TG      0x0
 #define SKIP_AGE_LOAD  0x0
 #define DO_ATTACH      0x0
+#define DO_DETACH      0x0
 
 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1)
 {
@@ -4512,6 +4521,11 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
 static void
 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
+       int action = UPDATE_TG;
+
+       if (entity_is_task(se) && task_on_rq_migrating(task_of(se)))
+               action |= DO_DETACH;
+
        /*
         * Update run-time statistics of the 'current'.
         */
@@ -4526,7 +4540,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
         *   - For group entity, update its weight to reflect the new share
         *     of its group cfs_rq.
         */
-       update_load_avg(cfs_rq, se, UPDATE_TG);
+       update_load_avg(cfs_rq, se, action);
        se_update_runnable(se);
 
        update_stats_dequeue_fair(cfs_rq, se, flags);
@@ -7078,8 +7092,6 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
        return new_cpu;
 }
 
-static void detach_entity_cfs_rq(struct sched_entity *se);
-
 /*
  * Called immediately before a task is migrated to a new CPU; task_cpu(p) and
  * cfs_rq_of(p) references at time of call are still valid and identify the
@@ -7101,15 +7113,7 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)
                se->vruntime -= u64_u32_load(cfs_rq->min_vruntime);
        }
 
-       if (p->on_rq == TASK_ON_RQ_MIGRATING) {
-               /*
-                * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old'
-                * rq->lock and can modify state directly.
-                */
-               lockdep_assert_rq_held(task_rq(p));
-               detach_entity_cfs_rq(se);
-
-       } else {
+       if (!task_on_rq_migrating(p)) {
                remove_entity_load_avg(se);
 
                /*