kernel/sched/isolation.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Housekeeping management. Manage the targets for routine code that can run on
   4  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   5  *
   6  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   7  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   8  *
   9  */
  10
  11 enum hk_flags {
  12         HK_FLAG_TIMER           = BIT(HK_TYPE_TIMER),
  13         HK_FLAG_RCU             = BIT(HK_TYPE_RCU),
  14         HK_FLAG_MISC            = BIT(HK_TYPE_MISC),
  15         HK_FLAG_SCHED           = BIT(HK_TYPE_SCHED),
  16         HK_FLAG_TICK            = BIT(HK_TYPE_TICK),
  17         HK_FLAG_DOMAIN          = BIT(HK_TYPE_DOMAIN),
  18         HK_FLAG_WQ              = BIT(HK_TYPE_WQ),
  19         HK_FLAG_MANAGED_IRQ     = BIT(HK_TYPE_MANAGED_IRQ),
  20         HK_FLAG_KTHREAD         = BIT(HK_TYPE_KTHREAD),
  21 };
  22
  23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  24 EXPORT_SYMBOL_GPL(housekeeping_overridden);
  25
  26 struct housekeeping {
  27         cpumask_var_t cpumasks[HK_TYPE_MAX];
  28         unsigned long flags;
  29 };
  30
  31 static struct housekeeping housekeeping;
  32
  33 bool housekeeping_enabled(enum hk_type type)
  34 {
  35         return !!(housekeeping.flags & BIT(type));
  36 }
  37 EXPORT_SYMBOL_GPL(housekeeping_enabled);
  38
  39 int housekeeping_any_cpu(enum hk_type type)
  40 {
  41         int cpu;
  42
  43         if (static_branch_unlikely(&housekeeping_overridden)) {
  44                 if (housekeeping.flags & BIT(type)) {
  45                         cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
  46                         if (cpu < nr_cpu_ids)
  47                                 return cpu;
  48
  49                         return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
  50                 }
  51         }
  52         return smp_processor_id();
  53 }
  54 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  55
  56 const struct cpumask *housekeeping_cpumask(enum hk_type type)
  57 {
  58         if (static_branch_unlikely(&housekeeping_overridden))
  59                 if (housekeeping.flags & BIT(type))
  60                         return housekeeping.cpumasks[type];
  61         return cpu_possible_mask;
  62 }
  63 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  64
  65 void housekeeping_affine(struct task_struct *t, enum hk_type type)
  66 {
  67         if (static_branch_unlikely(&housekeeping_overridden))
  68                 if (housekeeping.flags & BIT(type))
  69                         set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
  70 }
  71 EXPORT_SYMBOL_GPL(housekeeping_affine);
  72
  73 bool housekeeping_test_cpu(int cpu, enum hk_type type)
  74 {
  75         if (static_branch_unlikely(&housekeeping_overridden))
  76                 if (housekeeping.flags & BIT(type))
  77                         return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
  78         return true;
  79 }
  80 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  81
  82 void __init housekeeping_init(void)
  83 {
  84         enum hk_type type;
  85
  86         if (!housekeeping.flags)
  87                 return;
  88
  89         static_branch_enable(&housekeeping_overridden);
  90
  91         if (housekeeping.flags & HK_FLAG_TICK)
  92                 sched_tick_offload_init();
  93
  94         for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
  95                 /* We need at least one CPU to handle housekeeping work */
  96                 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
  97         }
  98 }
  99
 100 static void __init housekeeping_setup_type(enum hk_type type,
 101                                            cpumask_var_t housekeeping_staging)
 102 {
 103
 104         alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
 105         cpumask_copy(housekeeping.cpumasks[type],
 106                      housekeeping_staging);
 107 }
 108
 109 static int __init housekeeping_setup(char *str, unsigned long flags)
 110 {
 111         cpumask_var_t non_housekeeping_mask, housekeeping_staging;
 112         int err = 0;
 113
 114         if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
 115                 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 116                         pr_warn("Housekeeping: nohz unsupported."
 117                                 " Build with CONFIG_NO_HZ_FULL\n");
 118                         return 0;
 119                 }
 120         }
 121
 122         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
 123         if (cpulist_parse(str, non_housekeeping_mask) < 0) {
 124                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
 125                 goto free_non_housekeeping_mask;
 126         }
 127
 128         alloc_bootmem_cpumask_var(&housekeeping_staging);
 129         cpumask_andnot(housekeeping_staging,
 130                        cpu_possible_mask, non_housekeeping_mask);
 131
 132         if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
 133                 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
 134                 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
 135                 if (!housekeeping.flags) {
 136                         pr_warn("Housekeeping: must include one present CPU, "
 137                                 "using boot CPU:%d\n", smp_processor_id());
 138                 }
 139         }
 140
 141         if (!housekeeping.flags) {
 142                 /* First setup call ("nohz_full=" or "isolcpus=") */
 143                 enum hk_type type;
 144
 145                 for_each_set_bit(type, &flags, HK_TYPE_MAX)
 146                         housekeeping_setup_type(type, housekeeping_staging);
 147         } else {
 148                 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
 149                 enum hk_type type;
 150                 unsigned long iter_flags = flags & housekeeping.flags;
 151
 152                 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
 153                         if (!cpumask_equal(housekeeping_staging,
 154                                            housekeeping.cpumasks[type])) {
 155                                 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
 156                                 goto free_housekeeping_staging;
 157                         }
 158                 }
 159
 160                 iter_flags = flags & ~housekeeping.flags;
 161
 162                 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
 163                         housekeeping_setup_type(type, housekeeping_staging);
 164         }
 165
 166         if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
 167                 tick_nohz_full_setup(non_housekeeping_mask);
 168
 169         housekeeping.flags |= flags;
 170         err = 1;
 171
 172 free_housekeeping_staging:
 173         free_bootmem_cpumask_var(housekeeping_staging);
 174 free_non_housekeeping_mask:
 175         free_bootmem_cpumask_var(non_housekeeping_mask);
 176
 177         return err;
 178 }
 179
 180 static int __init housekeeping_nohz_full_setup(char *str)
 181 {
 182         unsigned long flags;
 183
 184         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
 185                 HK_FLAG_MISC | HK_FLAG_KTHREAD;
 186
 187         return housekeeping_setup(str, flags);
 188 }
 189 __setup("nohz_full=", housekeeping_nohz_full_setup);
 190
 191 static int __init housekeeping_isolcpus_setup(char *str)
 192 {
 193         unsigned long flags = 0;
 194         bool illegal = false;
 195         char *par;
 196         int len;
 197
 198         while (isalpha(*str)) {
 199                 if (!strncmp(str, "nohz,", 5)) {
 200                         str += 5;
 201                         flags |= HK_FLAG_TICK;
 202                         continue;
 203                 }
 204
 205                 if (!strncmp(str, "domain,", 7)) {
 206                         str += 7;
 207                         flags |= HK_FLAG_DOMAIN;
 208                         continue;
 209                 }
 210
 211                 if (!strncmp(str, "managed_irq,", 12)) {
 212                         str += 12;
 213                         flags |= HK_FLAG_MANAGED_IRQ;
 214                         continue;
 215                 }
 216
 217                 /*
 218                  * Skip unknown sub-parameter and validate that it is not
 219                  * containing an invalid character.
 220                  */
 221                 for (par = str, len = 0; *str && *str != ','; str++, len++) {
 222                         if (!isalpha(*str) && *str != '_')
 223                                 illegal = true;
 224                 }
 225
 226                 if (illegal) {
 227                         pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
 228                         return 0;
 229                 }
 230
 231                 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
 232                 str++;
 233         }
 234
 235         /* Default behaviour for isolcpus without flags */
 236         if (!flags)
 237                 flags |= HK_FLAG_DOMAIN;
 238
 239         return housekeeping_setup(str, flags);
 240 }
 241 __setup("isolcpus=", housekeeping_isolcpus_setup);