sched/core: Add {EN,DE}QUEUE_NOCLOCK flags

Currently {en,de}queue_task() do an unconditional update_rq_clock().
However since we want to avoid duplicate updates, so that each
rq->lock section appears atomic in time, we need to be able to skip
these clock updates.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c5a514b..ce363bd 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -752,17 +752,23 @@ static void set_load_weight(struct task_struct *p)
 
 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
 {
-       update_rq_clock(rq);
+       if (!(flags & ENQUEUE_NOCLOCK))
+               update_rq_clock(rq);
+
        if (!(flags & ENQUEUE_RESTORE))
                sched_info_queued(rq, p);
+
        p->sched_class->enqueue_task(rq, p, flags);
 }
 
 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 {
-       update_rq_clock(rq);
+       if (!(flags & DEQUEUE_NOCLOCK))
+               update_rq_clock(rq);
+
        if (!(flags & DEQUEUE_SAVE))
                sched_info_dequeued(rq, p);
+
        p->sched_class->dequeue_task(rq, p, flags);
 }
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 7d4f693..de4b934 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1331,15 +1331,17 @@ extern const u32 sched_prio_to_wmult[40];
 #define DEQUEUE_SLEEP          0x01
 #define DEQUEUE_SAVE           0x02 /* matches ENQUEUE_RESTORE */
 #define DEQUEUE_MOVE           0x04 /* matches ENQUEUE_MOVE */
+#define DEQUEUE_NOCLOCK                0x08 /* matches ENQUEUE_NOCLOCK */
 
 #define ENQUEUE_WAKEUP         0x01
 #define ENQUEUE_RESTORE                0x02
 #define ENQUEUE_MOVE           0x04
+#define ENQUEUE_NOCLOCK                0x08
 
-#define ENQUEUE_HEAD           0x08
-#define ENQUEUE_REPLENISH      0x10
+#define ENQUEUE_HEAD           0x10
+#define ENQUEUE_REPLENISH      0x20
 #ifdef CONFIG_SMP
-#define ENQUEUE_MIGRATED       0x20
+#define ENQUEUE_MIGRATED       0x40
 #else
 #define ENQUEUE_MIGRATED       0x00
 #endif