kernel/torture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Common functions for in-kernel torture tests.
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Author: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *      Based on kernel/rcu/torture.c.
   9  */
  10
  11 #define pr_fmt(fmt) fmt
  12
  13 #include <linux/types.h>
  14 #include <linux/kernel.h>
  15 #include <linux/init.h>
  16 #include <linux/module.h>
  17 #include <linux/kthread.h>
  18 #include <linux/err.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/smp.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/sched.h>
  23 #include <linux/sched/clock.h>
  24 #include <linux/atomic.h>
  25 #include <linux/bitops.h>
  26 #include <linux/completion.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/percpu.h>
  29 #include <linux/notifier.h>
  30 #include <linux/reboot.h>
  31 #include <linux/freezer.h>
  32 #include <linux/cpu.h>
  33 #include <linux/delay.h>
  34 #include <linux/stat.h>
  35 #include <linux/slab.h>
  36 #include <linux/trace_clock.h>
  37 #include <linux/ktime.h>
  38 #include <asm/byteorder.h>
  39 #include <linux/torture.h>
  40 #include "rcu/rcu.h"
  41
  42 MODULE_LICENSE("GPL");
  43 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  44
  45 static bool disable_onoff_at_boot;
  46 module_param(disable_onoff_at_boot, bool, 0444);
  47
  48 static bool ftrace_dump_at_shutdown;
  49 module_param(ftrace_dump_at_shutdown, bool, 0444);
  50
  51 static int verbose_sleep_frequency;
  52 module_param(verbose_sleep_frequency, int, 0444);
  53
  54 static int verbose_sleep_duration = 1;
  55 module_param(verbose_sleep_duration, int, 0444);
  56
  57 static char *torture_type;
  58 static int verbose;
  59
  60 /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
  61 #define FULLSTOP_DONTSTOP 0     /* Normal operation. */
  62 #define FULLSTOP_SHUTDOWN 1     /* System shutdown with torture running. */
  63 #define FULLSTOP_RMMOD    2     /* Normal rmmod of torture. */
  64 static int fullstop = FULLSTOP_RMMOD;
  65 static DEFINE_MUTEX(fullstop_mutex);
  66
  67 static atomic_t verbose_sleep_counter;
  68
  69 /*
  70  * Sleep if needed from VERBOSE_TOROUT*().
  71  */
  72 void verbose_torout_sleep(void)
  73 {
  74         if (verbose_sleep_frequency > 0 &&
  75             verbose_sleep_duration > 0 &&
  76             !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency))
  77                 schedule_timeout_uninterruptible(verbose_sleep_duration);
  78 }
  79 EXPORT_SYMBOL_GPL(verbose_torout_sleep);
  80
  81 /*
  82  * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit
  83  * nanosecond random fuzz.  This function and its friends desynchronize
  84  * testing from the timer wheel.
  85  */
  86 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp)
  87 {
  88         ktime_t hto = baset_ns;
  89
  90         if (trsp)
  91                 hto += (torture_random(trsp) >> 3) % fuzzt_ns;
  92         set_current_state(TASK_UNINTERRUPTIBLE);
  93         return schedule_hrtimeout(&hto, HRTIMER_MODE_REL);
  94 }
  95 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns);
  96
  97 /*
  98  * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit
  99  * nanosecond (not microsecond!) random fuzz.
 100  */
 101 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp)
 102 {
 103         ktime_t baset_ns = baset_us * NSEC_PER_USEC;
 104
 105         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 106 }
 107 EXPORT_SYMBOL_GPL(torture_hrtimeout_us);
 108
 109 /*
 110  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
 111  * microsecond (not millisecond!) random fuzz.
 112  */
 113 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp)
 114 {
 115         ktime_t baset_ns = baset_ms * NSEC_PER_MSEC;
 116         u32 fuzzt_ns;
 117
 118         if ((u32)~0U / NSEC_PER_USEC < fuzzt_us)
 119                 fuzzt_ns = (u32)~0U;
 120         else
 121                 fuzzt_ns = fuzzt_us * NSEC_PER_USEC;
 122         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 123 }
 124 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms);
 125
 126 /*
 127  * Schedule a high-resolution-timer sleep in jiffies, with an
 128  * implied one-jiffy random fuzz.  This is intended to replace calls to
 129  * schedule_timeout_interruptible() and friends.
 130  */
 131 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp)
 132 {
 133         ktime_t baset_ns = jiffies_to_nsecs(baset_j);
 134
 135         return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp);
 136 }
 137 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies);
 138
 139 /*
 140  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
 141  * millisecond (not second!) random fuzz.
 142  */
 143 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp)
 144 {
 145         ktime_t baset_ns = baset_s * NSEC_PER_SEC;
 146         u32 fuzzt_ns;
 147
 148         if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms)
 149                 fuzzt_ns = (u32)~0U;
 150         else
 151                 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC;
 152         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 153 }
 154 EXPORT_SYMBOL_GPL(torture_hrtimeout_s);
 155
 156 #ifdef CONFIG_HOTPLUG_CPU
 157
 158 /*
 159  * Variables for online-offline handling.  Only present if CPU hotplug
 160  * is enabled, otherwise does nothing.
 161  */
 162
 163 static struct task_struct *onoff_task;
 164 static long onoff_holdoff;
 165 static long onoff_interval;
 166 static torture_ofl_func *onoff_f;
 167 static long n_offline_attempts;
 168 static long n_offline_successes;
 169 static unsigned long sum_offline;
 170 static int min_offline = -1;
 171 static int max_offline;
 172 static long n_online_attempts;
 173 static long n_online_successes;
 174 static unsigned long sum_online;
 175 static int min_online = -1;
 176 static int max_online;
 177
 178 static int torture_online_cpus = NR_CPUS;
 179
 180 /*
 181  * Some torture testing leverages confusion as to the number of online
 182  * CPUs.  This function returns the torture-testing view of this number,
 183  * which allows torture tests to load-balance appropriately.
 184  */
 185 int torture_num_online_cpus(void)
 186 {
 187         return READ_ONCE(torture_online_cpus);
 188 }
 189 EXPORT_SYMBOL_GPL(torture_num_online_cpus);
 190
 191 /*
 192  * Attempt to take a CPU offline.  Return false if the CPU is already
 193  * offline or if it is not subject to CPU-hotplug operations.  The
 194  * caller can detect other failures by looking at the statistics.
 195  */
 196 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
 197                      unsigned long *sum_offl, int *min_offl, int *max_offl)
 198 {
 199         unsigned long delta;
 200         int ret;
 201         char *s;
 202         unsigned long starttime;
 203
 204         if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 205                 return false;
 206         if (num_online_cpus() <= 1)
 207                 return false;  /* Can't offline the last CPU. */
 208
 209         if (verbose > 1)
 210                 pr_alert("%s" TORTURE_FLAG
 211                          "torture_onoff task: offlining %d\n",
 212                          torture_type, cpu);
 213         starttime = jiffies;
 214         (*n_offl_attempts)++;
 215         ret = remove_cpu(cpu);
 216         if (ret) {
 217                 s = "";
 218                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
 219                         // PCI probe frequently disables hotplug during boot.
 220                         (*n_offl_attempts)--;
 221                         s = " (-EBUSY forgiven during boot)";
 222                 }
 223                 if (verbose)
 224                         pr_alert("%s" TORTURE_FLAG
 225                                  "torture_onoff task: offline %d failed%s: errno %d\n",
 226                                  torture_type, cpu, s, ret);
 227         } else {
 228                 if (verbose > 1)
 229                         pr_alert("%s" TORTURE_FLAG
 230                                  "torture_onoff task: offlined %d\n",
 231                                  torture_type, cpu);
 232                 if (onoff_f)
 233                         onoff_f();
 234                 (*n_offl_successes)++;
 235                 delta = jiffies - starttime;
 236                 *sum_offl += delta;
 237                 if (*min_offl < 0) {
 238                         *min_offl = delta;
 239                         *max_offl = delta;
 240                 }
 241                 if (*min_offl > delta)
 242                         *min_offl = delta;
 243                 if (*max_offl < delta)
 244                         *max_offl = delta;
 245                 WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1);
 246                 WARN_ON_ONCE(torture_online_cpus <= 0);
 247         }
 248
 249         return true;
 250 }
 251 EXPORT_SYMBOL_GPL(torture_offline);
 252
 253 /*
 254  * Attempt to bring a CPU online.  Return false if the CPU is already
 255  * online or if it is not subject to CPU-hotplug operations.  The
 256  * caller can detect other failures by looking at the statistics.
 257  */
 258 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
 259                     unsigned long *sum_onl, int *min_onl, int *max_onl)
 260 {
 261         unsigned long delta;
 262         int ret;
 263         char *s;
 264         unsigned long starttime;
 265
 266         if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 267                 return false;
 268
 269         if (verbose > 1)
 270                 pr_alert("%s" TORTURE_FLAG
 271                          "torture_onoff task: onlining %d\n",
 272                          torture_type, cpu);
 273         starttime = jiffies;
 274         (*n_onl_attempts)++;
 275         ret = add_cpu(cpu);
 276         if (ret) {
 277                 s = "";
 278                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
 279                         // PCI probe frequently disables hotplug during boot.
 280                         (*n_onl_attempts)--;
 281                         s = " (-EBUSY forgiven during boot)";
 282                 }
 283                 if (verbose)
 284                         pr_alert("%s" TORTURE_FLAG
 285                                  "torture_onoff task: online %d failed%s: errno %d\n",
 286                                  torture_type, cpu, s, ret);
 287         } else {
 288                 if (verbose > 1)
 289                         pr_alert("%s" TORTURE_FLAG
 290                                  "torture_onoff task: onlined %d\n",
 291                                  torture_type, cpu);
 292                 (*n_onl_successes)++;
 293                 delta = jiffies - starttime;
 294                 *sum_onl += delta;
 295                 if (*min_onl < 0) {
 296                         *min_onl = delta;
 297                         *max_onl = delta;
 298                 }
 299                 if (*min_onl > delta)
 300                         *min_onl = delta;
 301                 if (*max_onl < delta)
 302                         *max_onl = delta;
 303                 WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1);
 304         }
 305
 306         return true;
 307 }
 308 EXPORT_SYMBOL_GPL(torture_online);
 309
 310 /*
 311  * Get everything online at the beginning and ends of tests.
 312  */
 313 static void torture_online_all(char *phase)
 314 {
 315         int cpu;
 316         int ret;
 317
 318         for_each_possible_cpu(cpu) {
 319                 if (cpu_online(cpu))
 320                         continue;
 321                 ret = add_cpu(cpu);
 322                 if (ret && verbose) {
 323                         pr_alert("%s" TORTURE_FLAG
 324                                  "%s: %s online %d: errno %d\n",
 325                                  __func__, phase, torture_type, cpu, ret);
 326                 }
 327         }
 328 }
 329
 330 /*
 331  * Execute random CPU-hotplug operations at the interval specified
 332  * by the onoff_interval.
 333  */
 334 static int
 335 torture_onoff(void *arg)
 336 {
 337         int cpu;
 338         int maxcpu = -1;
 339         DEFINE_TORTURE_RANDOM(rand);
 340
 341         VERBOSE_TOROUT_STRING("torture_onoff task started");
 342         for_each_online_cpu(cpu)
 343                 maxcpu = cpu;
 344         WARN_ON(maxcpu < 0);
 345         torture_online_all("Initial");
 346         if (maxcpu == 0) {
 347                 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
 348                 goto stop;
 349         }
 350
 351         if (onoff_holdoff > 0) {
 352                 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
 353                 schedule_timeout_interruptible(onoff_holdoff);
 354                 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
 355         }
 356         while (!torture_must_stop()) {
 357                 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
 358                         schedule_timeout_interruptible(HZ / 10);
 359                         continue;
 360                 }
 361                 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
 362                 if (!torture_offline(cpu,
 363                                      &n_offline_attempts, &n_offline_successes,
 364                                      &sum_offline, &min_offline, &max_offline))
 365                         torture_online(cpu,
 366                                        &n_online_attempts, &n_online_successes,
 367                                        &sum_online, &min_online, &max_online);
 368                 schedule_timeout_interruptible(onoff_interval);
 369         }
 370
 371 stop:
 372         torture_kthread_stopping("torture_onoff");
 373         torture_online_all("Final");
 374         return 0;
 375 }
 376
 377 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 378
 379 /*
 380  * Initiate online-offline handling.
 381  */
 382 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f)
 383 {
 384 #ifdef CONFIG_HOTPLUG_CPU
 385         onoff_holdoff = ooholdoff;
 386         onoff_interval = oointerval;
 387         onoff_f = f;
 388         if (onoff_interval <= 0)
 389                 return 0;
 390         return torture_create_kthread(torture_onoff, NULL, onoff_task);
 391 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 392         return 0;
 393 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
 394 }
 395 EXPORT_SYMBOL_GPL(torture_onoff_init);
 396
 397 /*
 398  * Clean up after online/offline testing.
 399  */
 400 static void torture_onoff_cleanup(void)
 401 {
 402 #ifdef CONFIG_HOTPLUG_CPU
 403         if (onoff_task == NULL)
 404                 return;
 405         VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
 406         kthread_stop(onoff_task);
 407         onoff_task = NULL;
 408 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 409 }
 410
 411 /*
 412  * Print online/offline testing statistics.
 413  */
 414 void torture_onoff_stats(void)
 415 {
 416 #ifdef CONFIG_HOTPLUG_CPU
 417         pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
 418                 n_online_successes, n_online_attempts,
 419                 n_offline_successes, n_offline_attempts,
 420                 min_online, max_online,
 421                 min_offline, max_offline,
 422                 sum_online, sum_offline, HZ);
 423 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 424 }
 425 EXPORT_SYMBOL_GPL(torture_onoff_stats);
 426
 427 /*
 428  * Were all the online/offline operations successful?
 429  */
 430 bool torture_onoff_failures(void)
 431 {
 432 #ifdef CONFIG_HOTPLUG_CPU
 433         return n_online_successes != n_online_attempts ||
 434                n_offline_successes != n_offline_attempts;
 435 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 436         return false;
 437 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
 438 }
 439 EXPORT_SYMBOL_GPL(torture_onoff_failures);
 440
 441 #define TORTURE_RANDOM_MULT     39916801  /* prime */
 442 #define TORTURE_RANDOM_ADD      479001701 /* prime */
 443 #define TORTURE_RANDOM_REFRESH  10000
 444
 445 /*
 446  * Crude but fast random-number generator.  Uses a linear congruential
 447  * generator, with occasional help from cpu_clock().
 448  */
 449 unsigned long
 450 torture_random(struct torture_random_state *trsp)
 451 {
 452         if (--trsp->trs_count < 0) {
 453                 trsp->trs_state += (unsigned long)local_clock();
 454                 trsp->trs_count = TORTURE_RANDOM_REFRESH;
 455         }
 456         trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
 457                 TORTURE_RANDOM_ADD;
 458         return swahw32(trsp->trs_state);
 459 }
 460 EXPORT_SYMBOL_GPL(torture_random);
 461
 462 /*
 463  * Variables for shuffling.  The idea is to ensure that each CPU stays
 464  * idle for an extended period to test interactions with dyntick idle,
 465  * as well as interactions with any per-CPU variables.
 466  */
 467 struct shuffle_task {
 468         struct list_head st_l;
 469         struct task_struct *st_t;
 470 };
 471
 472 static long shuffle_interval;   /* In jiffies. */
 473 static struct task_struct *shuffler_task;
 474 static cpumask_var_t shuffle_tmp_mask;
 475 static int shuffle_idle_cpu;    /* Force all torture tasks off this CPU */
 476 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
 477 static DEFINE_MUTEX(shuffle_task_mutex);
 478
 479 /*
 480  * Register a task to be shuffled.  If there is no memory, just splat
 481  * and don't bother registering.
 482  */
 483 void torture_shuffle_task_register(struct task_struct *tp)
 484 {
 485         struct shuffle_task *stp;
 486
 487         if (WARN_ON_ONCE(tp == NULL))
 488                 return;
 489         stp = kmalloc(sizeof(*stp), GFP_KERNEL);
 490         if (WARN_ON_ONCE(stp == NULL))
 491                 return;
 492         stp->st_t = tp;
 493         mutex_lock(&shuffle_task_mutex);
 494         list_add(&stp->st_l, &shuffle_task_list);
 495         mutex_unlock(&shuffle_task_mutex);
 496 }
 497 EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
 498
 499 /*
 500  * Unregister all tasks, for example, at the end of the torture run.
 501  */
 502 static void torture_shuffle_task_unregister_all(void)
 503 {
 504         struct shuffle_task *stp;
 505         struct shuffle_task *p;
 506
 507         mutex_lock(&shuffle_task_mutex);
 508         list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
 509                 list_del(&stp->st_l);
 510                 kfree(stp);
 511         }
 512         mutex_unlock(&shuffle_task_mutex);
 513 }
 514
 515 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle.
 516  * A special case is when shuffle_idle_cpu = -1, in which case we allow
 517  * the tasks to run on all CPUs.
 518  */
 519 static void torture_shuffle_tasks(void)
 520 {
 521         struct shuffle_task *stp;
 522
 523         cpumask_setall(shuffle_tmp_mask);
 524         cpus_read_lock();
 525
 526         /* No point in shuffling if there is only one online CPU (ex: UP) */
 527         if (num_online_cpus() == 1) {
 528                 cpus_read_unlock();
 529                 return;
 530         }
 531
 532         /* Advance to the next CPU.  Upon overflow, don't idle any CPUs. */
 533         shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
 534         if (shuffle_idle_cpu >= nr_cpu_ids)
 535                 shuffle_idle_cpu = -1;
 536         else
 537                 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
 538
 539         mutex_lock(&shuffle_task_mutex);
 540         list_for_each_entry(stp, &shuffle_task_list, st_l)
 541                 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
 542         mutex_unlock(&shuffle_task_mutex);
 543
 544         cpus_read_unlock();
 545 }
 546
 547 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 548  * system to become idle at a time and cut off its timer ticks. This is meant
 549  * to test the support for such tickless idle CPU in RCU.
 550  */
 551 static int torture_shuffle(void *arg)
 552 {
 553         VERBOSE_TOROUT_STRING("torture_shuffle task started");
 554         do {
 555                 schedule_timeout_interruptible(shuffle_interval);
 556                 torture_shuffle_tasks();
 557                 torture_shutdown_absorb("torture_shuffle");
 558         } while (!torture_must_stop());
 559         torture_kthread_stopping("torture_shuffle");
 560         return 0;
 561 }
 562
 563 /*
 564  * Start the shuffler, with shuffint in jiffies.
 565  */
 566 int torture_shuffle_init(long shuffint)
 567 {
 568         shuffle_interval = shuffint;
 569
 570         shuffle_idle_cpu = -1;
 571
 572         if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
 573                 TOROUT_ERRSTRING("Failed to alloc mask");
 574                 return -ENOMEM;
 575         }
 576
 577         /* Create the shuffler thread */
 578         return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
 579 }
 580 EXPORT_SYMBOL_GPL(torture_shuffle_init);
 581
 582 /*
 583  * Stop the shuffling.
 584  */
 585 static void torture_shuffle_cleanup(void)
 586 {
 587         torture_shuffle_task_unregister_all();
 588         if (shuffler_task) {
 589                 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
 590                 kthread_stop(shuffler_task);
 591                 free_cpumask_var(shuffle_tmp_mask);
 592         }
 593         shuffler_task = NULL;
 594 }
 595
 596 /*
 597  * Variables for auto-shutdown.  This allows "lights out" torture runs
 598  * to be fully scripted.
 599  */
 600 static struct task_struct *shutdown_task;
 601 static ktime_t shutdown_time;           /* time to system shutdown. */
 602 static void (*torture_shutdown_hook)(void);
 603
 604 /*
 605  * Absorb kthreads into a kernel function that won't return, so that
 606  * they won't ever access module text or data again.
 607  */
 608 void torture_shutdown_absorb(const char *title)
 609 {
 610         while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 611                 pr_notice("torture thread %s parking due to system shutdown\n",
 612                           title);
 613                 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
 614         }
 615 }
 616 EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
 617
 618 /*
 619  * Cause the torture test to shutdown the system after the test has
 620  * run for the time specified by the shutdown_secs parameter.
 621  */
 622 static int torture_shutdown(void *arg)
 623 {
 624         ktime_t ktime_snap;
 625
 626         VERBOSE_TOROUT_STRING("torture_shutdown task started");
 627         ktime_snap = ktime_get();
 628         while (ktime_before(ktime_snap, shutdown_time) &&
 629                !torture_must_stop()) {
 630                 if (verbose)
 631                         pr_alert("%s" TORTURE_FLAG
 632                                  "torture_shutdown task: %llu ms remaining\n",
 633                                  torture_type,
 634                                  ktime_ms_delta(shutdown_time, ktime_snap));
 635                 set_current_state(TASK_INTERRUPTIBLE);
 636                 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
 637                 ktime_snap = ktime_get();
 638         }
 639         if (torture_must_stop()) {
 640                 torture_kthread_stopping("torture_shutdown");
 641                 return 0;
 642         }
 643
 644         /* OK, shut down the system. */
 645
 646         VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
 647         shutdown_task = NULL;   /* Avoid self-kill deadlock. */
 648         if (torture_shutdown_hook)
 649                 torture_shutdown_hook();
 650         else
 651                 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
 652         if (ftrace_dump_at_shutdown)
 653                 rcu_ftrace_dump(DUMP_ALL);
 654         kernel_power_off();     /* Shut down the system. */
 655         return 0;
 656 }
 657
 658 /*
 659  * Start up the shutdown task.
 660  */
 661 int torture_shutdown_init(int ssecs, void (*cleanup)(void))
 662 {
 663         torture_shutdown_hook = cleanup;
 664         if (ssecs > 0) {
 665                 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
 666                 return torture_create_kthread(torture_shutdown, NULL,
 667                                              shutdown_task);
 668         }
 669         return 0;
 670 }
 671 EXPORT_SYMBOL_GPL(torture_shutdown_init);
 672
 673 /*
 674  * Detect and respond to a system shutdown.
 675  */
 676 static int torture_shutdown_notify(struct notifier_block *unused1,
 677                                    unsigned long unused2, void *unused3)
 678 {
 679         mutex_lock(&fullstop_mutex);
 680         if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
 681                 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
 682                 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
 683         } else {
 684                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 685         }
 686         mutex_unlock(&fullstop_mutex);
 687         return NOTIFY_DONE;
 688 }
 689
 690 static struct notifier_block torture_shutdown_nb = {
 691         .notifier_call = torture_shutdown_notify,
 692 };
 693
 694 /*
 695  * Shut down the shutdown task.  Say what???  Heh!  This can happen if
 696  * the torture module gets an rmmod before the shutdown time arrives.  ;-)
 697  */
 698 static void torture_shutdown_cleanup(void)
 699 {
 700         unregister_reboot_notifier(&torture_shutdown_nb);
 701         if (shutdown_task != NULL) {
 702                 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
 703                 kthread_stop(shutdown_task);
 704         }
 705         shutdown_task = NULL;
 706 }
 707
 708 /*
 709  * Variables for stuttering, which means to periodically pause and
 710  * restart testing in order to catch bugs that appear when load is
 711  * suddenly applied to or removed from the system.
 712  */
 713 static struct task_struct *stutter_task;
 714 static int stutter_pause_test;
 715 static int stutter;
 716 static int stutter_gap;
 717
 718 /*
 719  * Block until the stutter interval ends.  This must be called periodically
 720  * by all running kthreads that need to be subject to stuttering.
 721  */
 722 bool stutter_wait(const char *title)
 723 {
 724         unsigned int i = 0;
 725         bool ret = false;
 726         int spt;
 727
 728         cond_resched_tasks_rcu_qs();
 729         spt = READ_ONCE(stutter_pause_test);
 730         for (; spt; spt = READ_ONCE(stutter_pause_test)) {
 731                 if (!ret) {
 732                         sched_set_normal(current, MAX_NICE);
 733                         ret = true;
 734                 }
 735                 if (spt == 1) {
 736                         schedule_timeout_interruptible(1);
 737                 } else if (spt == 2) {
 738                         while (READ_ONCE(stutter_pause_test)) {
 739                                 if (!(i++ & 0xffff))
 740                                         torture_hrtimeout_us(10, 0, NULL);
 741                                 cond_resched();
 742                         }
 743                 } else {
 744                         schedule_timeout_interruptible(round_jiffies_relative(HZ));
 745                 }
 746                 torture_shutdown_absorb(title);
 747         }
 748         return ret;
 749 }
 750 EXPORT_SYMBOL_GPL(stutter_wait);
 751
 752 /*
 753  * Cause the torture test to "stutter", starting and stopping all
 754  * threads periodically.
 755  */
 756 static int torture_stutter(void *arg)
 757 {
 758         DEFINE_TORTURE_RANDOM(rand);
 759         int wtime;
 760
 761         VERBOSE_TOROUT_STRING("torture_stutter task started");
 762         do {
 763                 if (!torture_must_stop() && stutter > 1) {
 764                         wtime = stutter;
 765                         if (stutter > 2) {
 766                                 WRITE_ONCE(stutter_pause_test, 1);
 767                                 wtime = stutter - 3;
 768                                 torture_hrtimeout_jiffies(wtime, &rand);
 769                                 wtime = 2;
 770                         }
 771                         WRITE_ONCE(stutter_pause_test, 2);
 772                         torture_hrtimeout_jiffies(wtime, NULL);
 773                 }
 774                 WRITE_ONCE(stutter_pause_test, 0);
 775                 if (!torture_must_stop())
 776                         torture_hrtimeout_jiffies(stutter_gap, NULL);
 777                 torture_shutdown_absorb("torture_stutter");
 778         } while (!torture_must_stop());
 779         torture_kthread_stopping("torture_stutter");
 780         return 0;
 781 }
 782
 783 /*
 784  * Initialize and kick off the torture_stutter kthread.
 785  */
 786 int torture_stutter_init(const int s, const int sgap)
 787 {
 788         stutter = s;
 789         stutter_gap = sgap;
 790         return torture_create_kthread(torture_stutter, NULL, stutter_task);
 791 }
 792 EXPORT_SYMBOL_GPL(torture_stutter_init);
 793
 794 /*
 795  * Cleanup after the torture_stutter kthread.
 796  */
 797 static void torture_stutter_cleanup(void)
 798 {
 799         if (!stutter_task)
 800                 return;
 801         VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
 802         kthread_stop(stutter_task);
 803         stutter_task = NULL;
 804 }
 805
 806 /*
 807  * Initialize torture module.  Please note that this is -not- invoked via
 808  * the usual module_init() mechanism, but rather by an explicit call from
 809  * the client torture module.  This call must be paired with a later
 810  * torture_init_end().
 811  *
 812  * The runnable parameter points to a flag that controls whether or not
 813  * the test is currently runnable.  If there is no such flag, pass in NULL.
 814  */
 815 bool torture_init_begin(char *ttype, int v)
 816 {
 817         mutex_lock(&fullstop_mutex);
 818         if (torture_type != NULL) {
 819                 pr_alert("%s: Refusing %s init: %s running.\n",
 820                           __func__, ttype, torture_type);
 821                 pr_alert("%s: One torture test at a time!\n", __func__);
 822                 mutex_unlock(&fullstop_mutex);
 823                 return false;
 824         }
 825         torture_type = ttype;
 826         verbose = v;
 827         fullstop = FULLSTOP_DONTSTOP;
 828         return true;
 829 }
 830 EXPORT_SYMBOL_GPL(torture_init_begin);
 831
 832 /*
 833  * Tell the torture module that initialization is complete.
 834  */
 835 void torture_init_end(void)
 836 {
 837         mutex_unlock(&fullstop_mutex);
 838         register_reboot_notifier(&torture_shutdown_nb);
 839 }
 840 EXPORT_SYMBOL_GPL(torture_init_end);
 841
 842 /*
 843  * Clean up torture module.  Please note that this is -not- invoked via
 844  * the usual module_exit() mechanism, but rather by an explicit call from
 845  * the client torture module.  Returns true if a race with system shutdown
 846  * is detected, otherwise, all kthreads started by functions in this file
 847  * will be shut down.
 848  *
 849  * This must be called before the caller starts shutting down its own
 850  * kthreads.
 851  *
 852  * Both torture_cleanup_begin() and torture_cleanup_end() must be paired,
 853  * in order to correctly perform the cleanup. They are separated because
 854  * threads can still need to reference the torture_type type, thus nullify
 855  * only after completing all other relevant calls.
 856  */
 857 bool torture_cleanup_begin(void)
 858 {
 859         mutex_lock(&fullstop_mutex);
 860         if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 861                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 862                 mutex_unlock(&fullstop_mutex);
 863                 schedule_timeout_uninterruptible(10);
 864                 return true;
 865         }
 866         WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
 867         mutex_unlock(&fullstop_mutex);
 868         torture_shutdown_cleanup();
 869         torture_shuffle_cleanup();
 870         torture_stutter_cleanup();
 871         torture_onoff_cleanup();
 872         return false;
 873 }
 874 EXPORT_SYMBOL_GPL(torture_cleanup_begin);
 875
 876 void torture_cleanup_end(void)
 877 {
 878         mutex_lock(&fullstop_mutex);
 879         torture_type = NULL;
 880         mutex_unlock(&fullstop_mutex);
 881 }
 882 EXPORT_SYMBOL_GPL(torture_cleanup_end);
 883
 884 /*
 885  * Is it time for the current torture test to stop?
 886  */
 887 bool torture_must_stop(void)
 888 {
 889         return torture_must_stop_irq() || kthread_should_stop();
 890 }
 891 EXPORT_SYMBOL_GPL(torture_must_stop);
 892
 893 /*
 894  * Is it time for the current torture test to stop?  This is the irq-safe
 895  * version, hence no check for kthread_should_stop().
 896  */
 897 bool torture_must_stop_irq(void)
 898 {
 899         return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
 900 }
 901 EXPORT_SYMBOL_GPL(torture_must_stop_irq);
 902
 903 /*
 904  * Each kthread must wait for kthread_should_stop() before returning from
 905  * its top-level function, otherwise segfaults ensue.  This function
 906  * prints a "stopping" message and waits for kthread_should_stop(), and
 907  * should be called from all torture kthreads immediately prior to
 908  * returning.
 909  */
 910 void torture_kthread_stopping(char *title)
 911 {
 912         char buf[128];
 913
 914         snprintf(buf, sizeof(buf), "%s is stopping", title);
 915         VERBOSE_TOROUT_STRING(buf);
 916         while (!kthread_should_stop()) {
 917                 torture_shutdown_absorb(title);
 918                 schedule_timeout_uninterruptible(1);
 919         }
 920 }
 921 EXPORT_SYMBOL_GPL(torture_kthread_stopping);
 922
 923 /*
 924  * Create a generic torture kthread that is immediately runnable.  If you
 925  * need the kthread to be stopped so that you can do something to it before
 926  * it starts, you will need to open-code your own.
 927  */
 928 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
 929                             char *f, struct task_struct **tp)
 930 {
 931         int ret = 0;
 932
 933         VERBOSE_TOROUT_STRING(m);
 934         *tp = kthread_create(fn, arg, "%s", s);
 935         if (IS_ERR(*tp)) {
 936                 ret = PTR_ERR(*tp);
 937                 TOROUT_ERRSTRING(f);
 938                 *tp = NULL;
 939                 return ret;
 940         }
 941         wake_up_process(*tp);  // Process is sleeping, so ordering provided.
 942         torture_shuffle_task_register(*tp);
 943         return ret;
 944 }
 945 EXPORT_SYMBOL_GPL(_torture_create_kthread);
 946
 947 /*
 948  * Stop a generic kthread, emitting a message.
 949  */
 950 void _torture_stop_kthread(char *m, struct task_struct **tp)
 951 {
 952         if (*tp == NULL)
 953                 return;
 954         VERBOSE_TOROUT_STRING(m);
 955         kthread_stop(*tp);
 956         *tp = NULL;
 957 }
 958 EXPORT_SYMBOL_GPL(_torture_stop_kthread);