sched/isolation: Introduce housekeeping flags

Before we implement isolcpus under housekeeping, we need the isolation
features to be more finegrained. For example some people want NOHZ_FULL
without the full scheduler isolation, others want full scheduler
isolation without NOHZ_FULL.

So let's cut all these isolation features piecewise, at the risk of
overcutting it right now. We can still merge some flags later if they
always make sense together.

Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Chris Metcalf <cmetcalf@mellanox.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Luiz Capitulino <lcapitulino@redhat.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Wanpeng Li <kernellwp@gmail.com>
Link: http://lkml.kernel.org/r/1509072159-31808-9-git-send-email-frederic@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Lukasz Luba <l.luba@partner.samsung.com>

diff --git a/drivers/net/ethernet/tile/tilegx.c b/drivers/net/ethernet/tile/tilegx.c
index 27a3272dcd48f321276bfa19f8abd5e7a4b1c842..b3e5816a4678fa7238286e53a470624088e51c43 100644 (file)
--- a/drivers/net/ethernet/tile/tilegx.c
+++ b/drivers/net/ethernet/tile/tilegx.c
@@ -2270,8 +2270,8 @@ static int __init tile_net_init_module(void)
                tile_net_dev_init(name, mac);
 
        if (!network_cpus_init())
-               cpumask_and(&network_cpus_map, housekeeping_cpumask(),
-                           cpu_online_mask);
+               cpumask_and(&network_cpus_map,
+                           housekeeping_cpumask(HK_FLAG_MISC), cpu_online_mask);
 
        return 0;
 }

diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h
index 93ac2367a520cec2eb098d289a756284736364ce..9bb753eece3b7e87d5ca0e94c9a9aa2b5afb2fa2 100644 (file)
--- a/include/linux/sched/isolation.h
+++ b/include/linux/sched/isolation.h
@@ -5,35 +5,43 @@
 #include <linux/init.h>
 #include <linux/tick.h>
 
+enum hk_flags {
+       HK_FLAG_TIMER           = 1,
+       HK_FLAG_RCU             = (1 << 1),
+       HK_FLAG_MISC            = (1 << 2),
+       HK_FLAG_SCHED           = (1 << 3),
+};
+
 #ifdef CONFIG_CPU_ISOLATION
 DECLARE_STATIC_KEY_FALSE(housekeeping_overriden);
-extern int housekeeping_any_cpu(void);
-extern const struct cpumask *housekeeping_cpumask(void);
-extern void housekeeping_affine(struct task_struct *t);
-extern bool housekeeping_test_cpu(int cpu);
+extern int housekeeping_any_cpu(enum hk_flags flags);
+extern const struct cpumask *housekeeping_cpumask(enum hk_flags flags);
+extern void housekeeping_affine(struct task_struct *t, enum hk_flags flags);
+extern bool housekeeping_test_cpu(int cpu, enum hk_flags flags);
 extern void __init housekeeping_init(void);
 
 #else
 
-static inline int housekeeping_any_cpu(void)
+static inline int housekeeping_any_cpu(enum hk_flags flags)
 {
        return smp_processor_id();
 }
 
-static inline const struct cpumask *housekeeping_cpumask(void)
+static inline const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
 {
        return cpu_possible_mask;
 }
 
-static inline void housekeeping_affine(struct task_struct *t) { }
+static inline void housekeeping_affine(struct task_struct *t,
+                                      enum hk_flags flags) { }
 static inline void housekeeping_init(void) { }
 #endif /* CONFIG_CPU_ISOLATION */
 
-static inline bool housekeeping_cpu(int cpu)
+static inline bool housekeeping_cpu(int cpu, enum hk_flags flags)
 {
 #ifdef CONFIG_CPU_ISOLATION
        if (static_branch_unlikely(&housekeeping_overriden))
-               return housekeeping_test_cpu(cpu);
+               return housekeeping_test_cpu(cpu, flags);
 #endif
        return true;
 }

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 34d4a563bb390a4885f43ce3c19e220fff606942..93d3c319f6198b50b157e01534df490929fdcf30 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2584,7 +2584,7 @@ static void rcu_bind_gp_kthread(void)
 
        if (!tick_nohz_full_enabled())
                return;
-       housekeeping_affine(current);
+       housekeeping_affine(current, HK_FLAG_RCU);
 }
 
 /* Record the current task on dyntick-idle entry. */

diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index be59c8131593cd547e15be31eccbc51a5c5e8499..50fac2abf4ab6a42d9189529e04857a34065873d 100644 (file)
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -721,7 +721,7 @@ static int __noreturn rcu_tasks_kthread(void *arg)
        LIST_HEAD(rcu_tasks_holdouts);
 
        /* Run on housekeeping CPUs by default.  Sysadm can move if desired. */
-       housekeeping_affine(current);
+       housekeeping_affine(current, HK_FLAG_RCU);
 
        /*
         * Each pass through the following loop makes one check for

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3c34af009216c722a3dee724e5d43564d7f0e663..de6c341fafc963d2e49b21d5d28855c56f83675a 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -526,7 +526,7 @@ int get_nohz_timer_target(void)
        int i, cpu = smp_processor_id();
        struct sched_domain *sd;
 
-       if (!idle_cpu(cpu) && housekeeping_cpu(cpu))
+       if (!idle_cpu(cpu) && housekeeping_cpu(cpu, HK_FLAG_TIMER))
                return cpu;
 
        rcu_read_lock();
@@ -535,15 +535,15 @@ int get_nohz_timer_target(void)
                        if (cpu == i)
                                continue;
 
-                       if (!idle_cpu(i) && housekeeping_cpu(i)) {
+                       if (!idle_cpu(i) && housekeeping_cpu(i, HK_FLAG_TIMER)) {
                                cpu = i;
                                goto unlock;
                        }
                }
        }
 
-       if (!housekeeping_cpu(cpu))
-               cpu = housekeeping_any_cpu();
+       if (!housekeeping_cpu(cpu, HK_FLAG_TIMER))
+               cpu = housekeeping_any_cpu(HK_FLAG_TIMER);
 unlock:
        rcu_read_unlock();
        return cpu;

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index dddb0197faedfae9501b3ff1c56b5489593ceeee..0989676c50e92df396e8a4ef6ccf3545a1e057fe 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -9028,7 +9028,7 @@ void nohz_balance_enter_idle(int cpu)
                return;
 
        /* Spare idle load balancing on CPUs that don't want to be disturbed: */
-       if (!housekeeping_cpu(cpu))
+       if (!housekeeping_cpu(cpu, HK_FLAG_SCHED))
                return;
 
        if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))

diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index bb8ba19a0235b2423ac46945d279a083fbe72a85..37a138a780ff2fe03fef5d2d02cd9026b9d713c3 100644 (file)
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -15,37 +15,39 @@
 DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
 EXPORT_SYMBOL_GPL(housekeeping_overriden);
 static cpumask_var_t housekeeping_mask;
+static unsigned int housekeeping_flags;
 
-int housekeeping_any_cpu(void)
+int housekeeping_any_cpu(enum hk_flags flags)
 {
        if (static_branch_unlikely(&housekeeping_overriden))
-               return cpumask_any_and(housekeeping_mask, cpu_online_mask);
-
+               if (housekeeping_flags & flags)
+                       return cpumask_any_and(housekeeping_mask, cpu_online_mask);
        return smp_processor_id();
 }
 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
 
-const struct cpumask *housekeeping_cpumask(void)
+const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
 {
        if (static_branch_unlikely(&housekeeping_overriden))
-               return housekeeping_mask;
-
+               if (housekeeping_flags & flags)
+                       return housekeeping_mask;
        return cpu_possible_mask;
 }
 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
 
-void housekeeping_affine(struct task_struct *t)
+void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
 {
        if (static_branch_unlikely(&housekeeping_overriden))
-               set_cpus_allowed_ptr(t, housekeeping_mask);
+               if (housekeeping_flags & flags)
+                       set_cpus_allowed_ptr(t, housekeeping_mask);
 }
 EXPORT_SYMBOL_GPL(housekeeping_affine);
 
-bool housekeeping_test_cpu(int cpu)
+bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
 {
        if (static_branch_unlikely(&housekeeping_overriden))
-               return cpumask_test_cpu(cpu, housekeeping_mask);
-
+               if (housekeeping_flags & flags)
+                       return cpumask_test_cpu(cpu, housekeeping_mask);
        return true;
 }
 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
@@ -65,6 +67,8 @@ void __init housekeeping_init(void)
        cpumask_andnot(housekeeping_mask,
                       cpu_possible_mask, tick_nohz_full_mask);
 
+       housekeeping_flags = HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
+
        static_branch_enable(&housekeeping_overriden);
 
        /* We need at least one CPU to handle housekeeping work */

diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index da20c8ad4d077d9d167d74f8db4d2d6cbcb90e88..576d1804581150d1479730166910ff6fb0a89d2e 100644 (file)
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -778,7 +778,8 @@ void __init lockup_detector_init(void)
        if (tick_nohz_full_enabled())
                pr_info("Disabling watchdog on nohz_full cores by default\n");
 
-       cpumask_copy(&watchdog_cpumask, housekeeping_cpumask());
+       cpumask_copy(&watchdog_cpumask,
+                    housekeeping_cpumask(HK_FLAG_TIMER));
 
        if (!watchdog_nmi_probe())
                nmi_watchdog_available = true;