Merge branch 'for-4.15' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup

[platform/kernel/linux-starfive.git] / kernel / sched / core.c

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0b3eec3..a092f35 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -26,6 +26,7 @@
 #include <linux/profile.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>
+#include <linux/sched/isolation.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -42,18 +43,21 @@
 
 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 
+#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
 /*
  * Debugging: various feature bits
+ *
+ * If SCHED_DEBUG is disabled, each compilation unit has its own copy of
+ * sysctl_sched_features, defined in sched.h, to allow constants propagation
+ * at compile time and compiler optimization based on features default.
  */
-
 #define SCHED_FEAT(name, enabled)      \
        (1UL << __SCHED_FEAT_##name) * enabled |
-
 const_debug unsigned int sysctl_sched_features =
 #include "features.h"
        0;
-
 #undef SCHED_FEAT
+#endif
 
 /*
  * Number of tasks to iterate in a single balance run.
@@ -83,9 +87,6 @@ __read_mostly int scheduler_running;
  */
 int sysctl_sched_rt_runtime = 950000;
 
-/* CPUs with isolated domains */
-cpumask_var_t cpu_isolated_map;
-
 /*
  * __task_rq_lock - lock the rq @p resides on.
  */
@@ -505,8 +506,7 @@ void resched_cpu(int cpu)
        struct rq *rq = cpu_rq(cpu);
        unsigned long flags;
 
-       if (!raw_spin_trylock_irqsave(&rq->lock, flags))
-               return;
+       raw_spin_lock_irqsave(&rq->lock, flags);
        resched_curr(rq);
        raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
@@ -526,7 +526,7 @@ int get_nohz_timer_target(void)
        int i, cpu = smp_processor_id();
        struct sched_domain *sd;
 
-       if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
+       if (!idle_cpu(cpu) && housekeeping_cpu(cpu, HK_FLAG_TIMER))
                return cpu;
 
        rcu_read_lock();
@@ -535,15 +535,15 @@ int get_nohz_timer_target(void)
                        if (cpu == i)
                                continue;
 
-                       if (!idle_cpu(i) && is_housekeeping_cpu(i)) {
+                       if (!idle_cpu(i) && housekeeping_cpu(i, HK_FLAG_TIMER)) {
                                cpu = i;
                                goto unlock;
                        }
                }
        }
 
-       if (!is_housekeeping_cpu(cpu))
-               cpu = housekeeping_any_cpu();
+       if (!housekeeping_cpu(cpu, HK_FLAG_TIMER))
+               cpu = housekeeping_any_cpu(HK_FLAG_TIMER);
 unlock:
        rcu_read_unlock();
        return cpu;
@@ -733,7 +733,7 @@ int tg_nop(struct task_group *tg, void *data)
 }
 #endif
 
-static void set_load_weight(struct task_struct *p)
+static void set_load_weight(struct task_struct *p, bool update_load)
 {
        int prio = p->static_prio - MAX_RT_PRIO;
        struct load_weight *load = &p->se.load;
@@ -747,8 +747,16 @@ static void set_load_weight(struct task_struct *p)
                return;
        }
 
-       load->weight = scale_load(sched_prio_to_weight[prio]);
-       load->inv_weight = sched_prio_to_wmult[prio];
+       /*
+        * SCHED_OTHER tasks have to update their load when changing their
+        * weight
+        */
+       if (update_load && p->sched_class == &fair_sched_class) {
+               reweight_task(p, prio);
+       } else {
+               load->weight = scale_load(sched_prio_to_weight[prio]);
+               load->inv_weight = sched_prio_to_wmult[prio];
+       }
 }
 
 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
@@ -2358,7 +2366,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
                        p->static_prio = NICE_TO_PRIO(0);
 
                p->prio = p->normal_prio = __normal_prio(p);
-               set_load_weight(p);
+               set_load_weight(p, false);
 
                /*
                 * We don't need the reset flag anymore after the fork. It has
@@ -3805,7 +3813,7 @@ void set_user_nice(struct task_struct *p, long nice)
                put_prev_task(rq, p);
 
        p->static_prio = NICE_TO_PRIO(nice);
-       set_load_weight(p);
+       set_load_weight(p, true);
        old_prio = p->prio;
        p->prio = effective_prio(p);
        delta = p->prio - old_prio;
@@ -3962,7 +3970,7 @@ static void __setscheduler_params(struct task_struct *p,
         */
        p->rt_priority = attr->sched_priority;
        p->normal_prio = normal_prio(p);
-       set_load_weight(p);
+       set_load_weight(p, true);
 }
 
 /* Actually do priority change: must hold pi & rq lock. */
@@ -4842,6 +4850,7 @@ int __sched _cond_resched(void)
                preempt_schedule_common();
                return 1;
        }
+       rcu_all_qs();
        return 0;
 }
 EXPORT_SYMBOL(_cond_resched);
@@ -5165,6 +5174,29 @@ void sched_show_task(struct task_struct *p)
        show_stack(p, NULL);
        put_task_stack(p);
 }
+EXPORT_SYMBOL_GPL(sched_show_task);
+
+static inline bool
+state_filter_match(unsigned long state_filter, struct task_struct *p)
+{
+       /* no filter, everything matches */
+       if (!state_filter)
+               return true;
+
+       /* filter, but doesn't match */
+       if (!(p->state & state_filter))
+               return false;
+
+       /*
+        * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
+        * TASK_KILLABLE).
+        */
+       if (state_filter == TASK_UNINTERRUPTIBLE && p->state == TASK_IDLE)
+               return false;
+
+       return true;
+}
+
 
 void show_state_filter(unsigned long state_filter)
 {
@@ -5188,7 +5220,7 @@ void show_state_filter(unsigned long state_filter)
                 */
                touch_nmi_watchdog();
                touch_all_softlockup_watchdogs();
-               if (!state_filter || (p->state & state_filter))
+               if (state_filter_match(state_filter, p))
                        sched_show_task(p);
        }
 
@@ -5704,10 +5736,6 @@ static inline void sched_init_smt(void) { }
 
 void __init sched_init_smp(void)
 {
-       cpumask_var_t non_isolated_cpus;
-
-       alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
-
        sched_init_numa();
 
        /*
@@ -5717,16 +5745,12 @@ void __init sched_init_smp(void)
         */
        mutex_lock(&sched_domains_mutex);
        sched_init_domains(cpu_active_mask);
-       cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
-       if (cpumask_empty(non_isolated_cpus))
-               cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
        mutex_unlock(&sched_domains_mutex);
 
        /* Move init over to a non-isolated CPU */
-       if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
+       if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0)
                BUG();
        sched_init_granularity();
-       free_cpumask_var(non_isolated_cpus);
 
        init_sched_rt_class();
        init_sched_dl_class();
@@ -5911,7 +5935,7 @@ void __init sched_init(void)
                atomic_set(&rq->nr_iowait, 0);
        }
 
-       set_load_weight(&init_task);
+       set_load_weight(&init_task, false);
 
        /*
         * The boot idle thread does lazy MMU switching as well:
@@ -5930,9 +5954,6 @@ void __init sched_init(void)
        calc_load_update = jiffies + LOAD_FREQ;
 
 #ifdef CONFIG_SMP
-       /* May be allocated at isolcpus cmdline parse time */
-       if (cpu_isolated_map == NULL)
-               zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
        idle_thread_set_boot_cpu();
        set_cpu_rq_start_time(smp_processor_id());
 #endif