kernel/torture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Common functions for in-kernel torture tests.
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Author: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *      Based on kernel/rcu/torture.c.
   9  */
  10
  11 #define pr_fmt(fmt) fmt
  12
  13 #include <linux/types.h>
  14 #include <linux/kernel.h>
  15 #include <linux/init.h>
  16 #include <linux/module.h>
  17 #include <linux/kthread.h>
  18 #include <linux/err.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/smp.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/sched.h>
  23 #include <linux/sched/clock.h>
  24 #include <linux/atomic.h>
  25 #include <linux/bitops.h>
  26 #include <linux/completion.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/percpu.h>
  29 #include <linux/notifier.h>
  30 #include <linux/reboot.h>
  31 #include <linux/freezer.h>
  32 #include <linux/cpu.h>
  33 #include <linux/delay.h>
  34 #include <linux/stat.h>
  35 #include <linux/slab.h>
  36 #include <linux/trace_clock.h>
  37 #include <linux/ktime.h>
  38 #include <asm/byteorder.h>
  39 #include <linux/torture.h>
  40 #include <linux/sched/rt.h>
  41 #include "rcu/rcu.h"
  42
  43 MODULE_LICENSE("GPL");
  44 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  45
  46 static bool disable_onoff_at_boot;
  47 module_param(disable_onoff_at_boot, bool, 0444);
  48
  49 static bool ftrace_dump_at_shutdown;
  50 module_param(ftrace_dump_at_shutdown, bool, 0444);
  51
  52 static int verbose_sleep_frequency;
  53 module_param(verbose_sleep_frequency, int, 0444);
  54
  55 static int verbose_sleep_duration = 1;
  56 module_param(verbose_sleep_duration, int, 0444);
  57
  58 static int random_shuffle;
  59 module_param(random_shuffle, int, 0444);
  60
  61 static char *torture_type;
  62 static int verbose;
  63
  64 /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
  65 #define FULLSTOP_DONTSTOP 0     /* Normal operation. */
  66 #define FULLSTOP_SHUTDOWN 1     /* System shutdown with torture running. */
  67 #define FULLSTOP_RMMOD    2     /* Normal rmmod of torture. */
  68 static int fullstop = FULLSTOP_RMMOD;
  69 static DEFINE_MUTEX(fullstop_mutex);
  70
  71 static atomic_t verbose_sleep_counter;
  72
  73 /*
  74  * Sleep if needed from VERBOSE_TOROUT*().
  75  */
  76 void verbose_torout_sleep(void)
  77 {
  78         if (verbose_sleep_frequency > 0 &&
  79             verbose_sleep_duration > 0 &&
  80             !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency))
  81                 schedule_timeout_uninterruptible(verbose_sleep_duration);
  82 }
  83 EXPORT_SYMBOL_GPL(verbose_torout_sleep);
  84
  85 /*
  86  * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit
  87  * nanosecond random fuzz.  This function and its friends desynchronize
  88  * testing from the timer wheel.
  89  */
  90 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp)
  91 {
  92         ktime_t hto = baset_ns;
  93
  94         if (trsp)
  95                 hto += torture_random(trsp) % fuzzt_ns;
  96         set_current_state(TASK_IDLE);
  97         return schedule_hrtimeout(&hto, HRTIMER_MODE_REL);
  98 }
  99 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns);
 100
 101 /*
 102  * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit
 103  * nanosecond (not microsecond!) random fuzz.
 104  */
 105 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp)
 106 {
 107         ktime_t baset_ns = baset_us * NSEC_PER_USEC;
 108
 109         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 110 }
 111 EXPORT_SYMBOL_GPL(torture_hrtimeout_us);
 112
 113 /*
 114  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
 115  * microsecond (not millisecond!) random fuzz.
 116  */
 117 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp)
 118 {
 119         ktime_t baset_ns = baset_ms * NSEC_PER_MSEC;
 120         u32 fuzzt_ns;
 121
 122         if ((u32)~0U / NSEC_PER_USEC < fuzzt_us)
 123                 fuzzt_ns = (u32)~0U;
 124         else
 125                 fuzzt_ns = fuzzt_us * NSEC_PER_USEC;
 126         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 127 }
 128 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms);
 129
 130 /*
 131  * Schedule a high-resolution-timer sleep in jiffies, with an
 132  * implied one-jiffy random fuzz.  This is intended to replace calls to
 133  * schedule_timeout_interruptible() and friends.
 134  */
 135 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp)
 136 {
 137         ktime_t baset_ns = jiffies_to_nsecs(baset_j);
 138
 139         return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp);
 140 }
 141 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies);
 142
 143 /*
 144  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
 145  * millisecond (not second!) random fuzz.
 146  */
 147 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp)
 148 {
 149         ktime_t baset_ns = baset_s * NSEC_PER_SEC;
 150         u32 fuzzt_ns;
 151
 152         if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms)
 153                 fuzzt_ns = (u32)~0U;
 154         else
 155                 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC;
 156         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 157 }
 158 EXPORT_SYMBOL_GPL(torture_hrtimeout_s);
 159
 160 #ifdef CONFIG_HOTPLUG_CPU
 161
 162 /*
 163  * Variables for online-offline handling.  Only present if CPU hotplug
 164  * is enabled, otherwise does nothing.
 165  */
 166
 167 static struct task_struct *onoff_task;
 168 static long onoff_holdoff;
 169 static long onoff_interval;
 170 static torture_ofl_func *onoff_f;
 171 static long n_offline_attempts;
 172 static long n_offline_successes;
 173 static unsigned long sum_offline;
 174 static int min_offline = -1;
 175 static int max_offline;
 176 static long n_online_attempts;
 177 static long n_online_successes;
 178 static unsigned long sum_online;
 179 static int min_online = -1;
 180 static int max_online;
 181
 182 static int torture_online_cpus = NR_CPUS;
 183
 184 /*
 185  * Some torture testing leverages confusion as to the number of online
 186  * CPUs.  This function returns the torture-testing view of this number,
 187  * which allows torture tests to load-balance appropriately.
 188  */
 189 int torture_num_online_cpus(void)
 190 {
 191         return READ_ONCE(torture_online_cpus);
 192 }
 193 EXPORT_SYMBOL_GPL(torture_num_online_cpus);
 194
 195 /*
 196  * Attempt to take a CPU offline.  Return false if the CPU is already
 197  * offline or if it is not subject to CPU-hotplug operations.  The
 198  * caller can detect other failures by looking at the statistics.
 199  */
 200 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
 201                      unsigned long *sum_offl, int *min_offl, int *max_offl)
 202 {
 203         unsigned long delta;
 204         int ret;
 205         char *s;
 206         unsigned long starttime;
 207
 208         if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 209                 return false;
 210         if (num_online_cpus() <= 1)
 211                 return false;  /* Can't offline the last CPU. */
 212
 213         if (verbose > 1)
 214                 pr_alert("%s" TORTURE_FLAG
 215                          "torture_onoff task: offlining %d\n",
 216                          torture_type, cpu);
 217         starttime = jiffies;
 218         (*n_offl_attempts)++;
 219         ret = remove_cpu(cpu);
 220         if (ret) {
 221                 s = "";
 222                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
 223                         // PCI probe frequently disables hotplug during boot.
 224                         (*n_offl_attempts)--;
 225                         s = " (-EBUSY forgiven during boot)";
 226                 }
 227                 if (verbose)
 228                         pr_alert("%s" TORTURE_FLAG
 229                                  "torture_onoff task: offline %d failed%s: errno %d\n",
 230                                  torture_type, cpu, s, ret);
 231         } else {
 232                 if (verbose > 1)
 233                         pr_alert("%s" TORTURE_FLAG
 234                                  "torture_onoff task: offlined %d\n",
 235                                  torture_type, cpu);
 236                 if (onoff_f)
 237                         onoff_f();
 238                 (*n_offl_successes)++;
 239                 delta = jiffies - starttime;
 240                 *sum_offl += delta;
 241                 if (*min_offl < 0) {
 242                         *min_offl = delta;
 243                         *max_offl = delta;
 244                 }
 245                 if (*min_offl > delta)
 246                         *min_offl = delta;
 247                 if (*max_offl < delta)
 248                         *max_offl = delta;
 249                 WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1);
 250                 WARN_ON_ONCE(torture_online_cpus <= 0);
 251         }
 252
 253         return true;
 254 }
 255 EXPORT_SYMBOL_GPL(torture_offline);
 256
 257 /*
 258  * Attempt to bring a CPU online.  Return false if the CPU is already
 259  * online or if it is not subject to CPU-hotplug operations.  The
 260  * caller can detect other failures by looking at the statistics.
 261  */
 262 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
 263                     unsigned long *sum_onl, int *min_onl, int *max_onl)
 264 {
 265         unsigned long delta;
 266         int ret;
 267         char *s;
 268         unsigned long starttime;
 269
 270         if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 271                 return false;
 272
 273         if (verbose > 1)
 274                 pr_alert("%s" TORTURE_FLAG
 275                          "torture_onoff task: onlining %d\n",
 276                          torture_type, cpu);
 277         starttime = jiffies;
 278         (*n_onl_attempts)++;
 279         ret = add_cpu(cpu);
 280         if (ret) {
 281                 s = "";
 282                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
 283                         // PCI probe frequently disables hotplug during boot.
 284                         (*n_onl_attempts)--;
 285                         s = " (-EBUSY forgiven during boot)";
 286                 }
 287                 if (verbose)
 288                         pr_alert("%s" TORTURE_FLAG
 289                                  "torture_onoff task: online %d failed%s: errno %d\n",
 290                                  torture_type, cpu, s, ret);
 291         } else {
 292                 if (verbose > 1)
 293                         pr_alert("%s" TORTURE_FLAG
 294                                  "torture_onoff task: onlined %d\n",
 295                                  torture_type, cpu);
 296                 (*n_onl_successes)++;
 297                 delta = jiffies - starttime;
 298                 *sum_onl += delta;
 299                 if (*min_onl < 0) {
 300                         *min_onl = delta;
 301                         *max_onl = delta;
 302                 }
 303                 if (*min_onl > delta)
 304                         *min_onl = delta;
 305                 if (*max_onl < delta)
 306                         *max_onl = delta;
 307                 WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1);
 308         }
 309
 310         return true;
 311 }
 312 EXPORT_SYMBOL_GPL(torture_online);
 313
 314 /*
 315  * Get everything online at the beginning and ends of tests.
 316  */
 317 static void torture_online_all(char *phase)
 318 {
 319         int cpu;
 320         int ret;
 321
 322         for_each_possible_cpu(cpu) {
 323                 if (cpu_online(cpu))
 324                         continue;
 325                 ret = add_cpu(cpu);
 326                 if (ret && verbose) {
 327                         pr_alert("%s" TORTURE_FLAG
 328                                  "%s: %s online %d: errno %d\n",
 329                                  __func__, phase, torture_type, cpu, ret);
 330                 }
 331         }
 332 }
 333
 334 /*
 335  * Execute random CPU-hotplug operations at the interval specified
 336  * by the onoff_interval.
 337  */
 338 static int
 339 torture_onoff(void *arg)
 340 {
 341         int cpu;
 342         int maxcpu = -1;
 343         DEFINE_TORTURE_RANDOM(rand);
 344
 345         VERBOSE_TOROUT_STRING("torture_onoff task started");
 346         for_each_online_cpu(cpu)
 347                 maxcpu = cpu;
 348         WARN_ON(maxcpu < 0);
 349         torture_online_all("Initial");
 350         if (maxcpu == 0) {
 351                 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
 352                 goto stop;
 353         }
 354
 355         if (onoff_holdoff > 0) {
 356                 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
 357                 torture_hrtimeout_jiffies(onoff_holdoff, &rand);
 358                 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
 359         }
 360         while (!torture_must_stop()) {
 361                 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
 362                         torture_hrtimeout_jiffies(HZ / 10, &rand);
 363                         continue;
 364                 }
 365                 cpu = torture_random(&rand) % (maxcpu + 1);
 366                 if (!torture_offline(cpu,
 367                                      &n_offline_attempts, &n_offline_successes,
 368                                      &sum_offline, &min_offline, &max_offline))
 369                         torture_online(cpu,
 370                                        &n_online_attempts, &n_online_successes,
 371                                        &sum_online, &min_online, &max_online);
 372                 torture_hrtimeout_jiffies(onoff_interval, &rand);
 373         }
 374
 375 stop:
 376         torture_kthread_stopping("torture_onoff");
 377         torture_online_all("Final");
 378         return 0;
 379 }
 380
 381 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 382
 383 /*
 384  * Initiate online-offline handling.
 385  */
 386 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f)
 387 {
 388 #ifdef CONFIG_HOTPLUG_CPU
 389         onoff_holdoff = ooholdoff;
 390         onoff_interval = oointerval;
 391         onoff_f = f;
 392         if (onoff_interval <= 0)
 393                 return 0;
 394         return torture_create_kthread(torture_onoff, NULL, onoff_task);
 395 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 396         return 0;
 397 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
 398 }
 399 EXPORT_SYMBOL_GPL(torture_onoff_init);
 400
 401 /*
 402  * Clean up after online/offline testing.
 403  */
 404 static void torture_onoff_cleanup(void)
 405 {
 406 #ifdef CONFIG_HOTPLUG_CPU
 407         if (onoff_task == NULL)
 408                 return;
 409         VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
 410         kthread_stop(onoff_task);
 411         onoff_task = NULL;
 412 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 413 }
 414
 415 /*
 416  * Print online/offline testing statistics.
 417  */
 418 void torture_onoff_stats(void)
 419 {
 420 #ifdef CONFIG_HOTPLUG_CPU
 421         pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
 422                 n_online_successes, n_online_attempts,
 423                 n_offline_successes, n_offline_attempts,
 424                 min_online, max_online,
 425                 min_offline, max_offline,
 426                 sum_online, sum_offline, HZ);
 427 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 428 }
 429 EXPORT_SYMBOL_GPL(torture_onoff_stats);
 430
 431 /*
 432  * Were all the online/offline operations successful?
 433  */
 434 bool torture_onoff_failures(void)
 435 {
 436 #ifdef CONFIG_HOTPLUG_CPU
 437         return n_online_successes != n_online_attempts ||
 438                n_offline_successes != n_offline_attempts;
 439 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 440         return false;
 441 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
 442 }
 443 EXPORT_SYMBOL_GPL(torture_onoff_failures);
 444
 445 #define TORTURE_RANDOM_MULT     39916801  /* prime */
 446 #define TORTURE_RANDOM_ADD      479001701 /* prime */
 447 #define TORTURE_RANDOM_REFRESH  10000
 448
 449 /*
 450  * Crude but fast random-number generator.  Uses a linear congruential
 451  * generator, with occasional help from cpu_clock().
 452  */
 453 unsigned long
 454 torture_random(struct torture_random_state *trsp)
 455 {
 456         if (--trsp->trs_count < 0) {
 457                 trsp->trs_state += (unsigned long)local_clock() + raw_smp_processor_id();
 458                 trsp->trs_count = TORTURE_RANDOM_REFRESH;
 459         }
 460         trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
 461                 TORTURE_RANDOM_ADD;
 462         return swahw32(trsp->trs_state);
 463 }
 464 EXPORT_SYMBOL_GPL(torture_random);
 465
 466 /*
 467  * Variables for shuffling.  The idea is to ensure that each CPU stays
 468  * idle for an extended period to test interactions with dyntick idle,
 469  * as well as interactions with any per-CPU variables.
 470  */
 471 struct shuffle_task {
 472         struct list_head st_l;
 473         struct task_struct *st_t;
 474 };
 475
 476 static long shuffle_interval;   /* In jiffies. */
 477 static struct task_struct *shuffler_task;
 478 static cpumask_var_t shuffle_tmp_mask;
 479 static int shuffle_idle_cpu;    /* Force all torture tasks off this CPU */
 480 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
 481 static DEFINE_MUTEX(shuffle_task_mutex);
 482
 483 /*
 484  * Register a task to be shuffled.  If there is no memory, just splat
 485  * and don't bother registering.
 486  */
 487 void torture_shuffle_task_register(struct task_struct *tp)
 488 {
 489         struct shuffle_task *stp;
 490
 491         if (WARN_ON_ONCE(tp == NULL))
 492                 return;
 493         stp = kmalloc(sizeof(*stp), GFP_KERNEL);
 494         if (WARN_ON_ONCE(stp == NULL))
 495                 return;
 496         stp->st_t = tp;
 497         mutex_lock(&shuffle_task_mutex);
 498         list_add(&stp->st_l, &shuffle_task_list);
 499         mutex_unlock(&shuffle_task_mutex);
 500 }
 501 EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
 502
 503 /*
 504  * Unregister all tasks, for example, at the end of the torture run.
 505  */
 506 static void torture_shuffle_task_unregister_all(void)
 507 {
 508         struct shuffle_task *stp;
 509         struct shuffle_task *p;
 510
 511         mutex_lock(&shuffle_task_mutex);
 512         list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
 513                 list_del(&stp->st_l);
 514                 kfree(stp);
 515         }
 516         mutex_unlock(&shuffle_task_mutex);
 517 }
 518
 519 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle.
 520  * A special case is when shuffle_idle_cpu = -1, in which case we allow
 521  * the tasks to run on all CPUs.
 522  */
 523 static void torture_shuffle_tasks(void)
 524 {
 525         DEFINE_TORTURE_RANDOM(rand);
 526         struct shuffle_task *stp;
 527
 528         cpumask_setall(shuffle_tmp_mask);
 529         cpus_read_lock();
 530
 531         /* No point in shuffling if there is only one online CPU (ex: UP) */
 532         if (num_online_cpus() == 1) {
 533                 cpus_read_unlock();
 534                 return;
 535         }
 536
 537         /* Advance to the next CPU.  Upon overflow, don't idle any CPUs. */
 538         shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
 539         if (shuffle_idle_cpu >= nr_cpu_ids)
 540                 shuffle_idle_cpu = -1;
 541         else
 542                 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
 543
 544         mutex_lock(&shuffle_task_mutex);
 545         list_for_each_entry(stp, &shuffle_task_list, st_l) {
 546                 if (!random_shuffle || torture_random(&rand) & 0x1)
 547                         set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
 548         }
 549         mutex_unlock(&shuffle_task_mutex);
 550
 551         cpus_read_unlock();
 552 }
 553
 554 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 555  * system to become idle at a time and cut off its timer ticks. This is meant
 556  * to test the support for such tickless idle CPU in RCU.
 557  */
 558 static int torture_shuffle(void *arg)
 559 {
 560         DEFINE_TORTURE_RANDOM(rand);
 561
 562         VERBOSE_TOROUT_STRING("torture_shuffle task started");
 563         do {
 564                 torture_hrtimeout_jiffies(shuffle_interval, &rand);
 565                 torture_shuffle_tasks();
 566                 torture_shutdown_absorb("torture_shuffle");
 567         } while (!torture_must_stop());
 568         torture_kthread_stopping("torture_shuffle");
 569         return 0;
 570 }
 571
 572 /*
 573  * Start the shuffler, with shuffint in jiffies.
 574  */
 575 int torture_shuffle_init(long shuffint)
 576 {
 577         shuffle_interval = shuffint;
 578
 579         shuffle_idle_cpu = -1;
 580
 581         if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
 582                 TOROUT_ERRSTRING("Failed to alloc mask");
 583                 return -ENOMEM;
 584         }
 585
 586         /* Create the shuffler thread */
 587         return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
 588 }
 589 EXPORT_SYMBOL_GPL(torture_shuffle_init);
 590
 591 /*
 592  * Stop the shuffling.
 593  */
 594 static void torture_shuffle_cleanup(void)
 595 {
 596         torture_shuffle_task_unregister_all();
 597         if (shuffler_task) {
 598                 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
 599                 kthread_stop(shuffler_task);
 600                 free_cpumask_var(shuffle_tmp_mask);
 601         }
 602         shuffler_task = NULL;
 603 }
 604
 605 /*
 606  * Variables for auto-shutdown.  This allows "lights out" torture runs
 607  * to be fully scripted.
 608  */
 609 static struct task_struct *shutdown_task;
 610 static ktime_t shutdown_time;           /* time to system shutdown. */
 611 static void (*torture_shutdown_hook)(void);
 612
 613 /*
 614  * Absorb kthreads into a kernel function that won't return, so that
 615  * they won't ever access module text or data again.
 616  */
 617 void torture_shutdown_absorb(const char *title)
 618 {
 619         while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 620                 pr_notice("torture thread %s parking due to system shutdown\n",
 621                           title);
 622                 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
 623         }
 624 }
 625 EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
 626
 627 /*
 628  * Cause the torture test to shutdown the system after the test has
 629  * run for the time specified by the shutdown_secs parameter.
 630  */
 631 static int torture_shutdown(void *arg)
 632 {
 633         ktime_t ktime_snap;
 634
 635         VERBOSE_TOROUT_STRING("torture_shutdown task started");
 636         ktime_snap = ktime_get();
 637         while (ktime_before(ktime_snap, shutdown_time) &&
 638                !torture_must_stop()) {
 639                 if (verbose)
 640                         pr_alert("%s" TORTURE_FLAG
 641                                  "torture_shutdown task: %llu ms remaining\n",
 642                                  torture_type,
 643                                  ktime_ms_delta(shutdown_time, ktime_snap));
 644                 set_current_state(TASK_INTERRUPTIBLE);
 645                 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
 646                 ktime_snap = ktime_get();
 647         }
 648         if (torture_must_stop()) {
 649                 torture_kthread_stopping("torture_shutdown");
 650                 return 0;
 651         }
 652
 653         /* OK, shut down the system. */
 654
 655         VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
 656         shutdown_task = NULL;   /* Avoid self-kill deadlock. */
 657         if (torture_shutdown_hook)
 658                 torture_shutdown_hook();
 659         else
 660                 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
 661         if (ftrace_dump_at_shutdown)
 662                 rcu_ftrace_dump(DUMP_ALL);
 663         kernel_power_off();     /* Shut down the system. */
 664         return 0;
 665 }
 666
 667 /*
 668  * Start up the shutdown task.
 669  */
 670 int torture_shutdown_init(int ssecs, void (*cleanup)(void))
 671 {
 672         torture_shutdown_hook = cleanup;
 673         if (ssecs > 0) {
 674                 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
 675                 return torture_create_kthread(torture_shutdown, NULL,
 676                                              shutdown_task);
 677         }
 678         return 0;
 679 }
 680 EXPORT_SYMBOL_GPL(torture_shutdown_init);
 681
 682 /*
 683  * Detect and respond to a system shutdown.
 684  */
 685 static int torture_shutdown_notify(struct notifier_block *unused1,
 686                                    unsigned long unused2, void *unused3)
 687 {
 688         mutex_lock(&fullstop_mutex);
 689         if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
 690                 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
 691                 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
 692         } else {
 693                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 694         }
 695         mutex_unlock(&fullstop_mutex);
 696         return NOTIFY_DONE;
 697 }
 698
 699 static struct notifier_block torture_shutdown_nb = {
 700         .notifier_call = torture_shutdown_notify,
 701 };
 702
 703 /*
 704  * Shut down the shutdown task.  Say what???  Heh!  This can happen if
 705  * the torture module gets an rmmod before the shutdown time arrives.  ;-)
 706  */
 707 static void torture_shutdown_cleanup(void)
 708 {
 709         unregister_reboot_notifier(&torture_shutdown_nb);
 710         if (shutdown_task != NULL) {
 711                 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
 712                 kthread_stop(shutdown_task);
 713         }
 714         shutdown_task = NULL;
 715 }
 716
 717 /*
 718  * Variables for stuttering, which means to periodically pause and
 719  * restart testing in order to catch bugs that appear when load is
 720  * suddenly applied to or removed from the system.
 721  */
 722 static struct task_struct *stutter_task;
 723 static int stutter_pause_test;
 724 static int stutter;
 725 static int stutter_gap;
 726
 727 /*
 728  * Block until the stutter interval ends.  This must be called periodically
 729  * by all running kthreads that need to be subject to stuttering.
 730  */
 731 bool stutter_wait(const char *title)
 732 {
 733         unsigned int i = 0;
 734         bool ret = false;
 735         int spt;
 736
 737         cond_resched_tasks_rcu_qs();
 738         spt = READ_ONCE(stutter_pause_test);
 739         for (; spt; spt = READ_ONCE(stutter_pause_test)) {
 740                 if (!ret && !rt_task(current)) {
 741                         sched_set_normal(current, MAX_NICE);
 742                         ret = true;
 743                 }
 744                 if (spt == 1) {
 745                         torture_hrtimeout_jiffies(1, NULL);
 746                 } else if (spt == 2) {
 747                         while (READ_ONCE(stutter_pause_test)) {
 748                                 if (!(i++ & 0xffff))
 749                                         torture_hrtimeout_us(10, 0, NULL);
 750                                 cond_resched();
 751                         }
 752                 } else {
 753                         torture_hrtimeout_jiffies(round_jiffies_relative(HZ), NULL);
 754                 }
 755                 torture_shutdown_absorb(title);
 756         }
 757         return ret;
 758 }
 759 EXPORT_SYMBOL_GPL(stutter_wait);
 760
 761 /*
 762  * Cause the torture test to "stutter", starting and stopping all
 763  * threads periodically.
 764  */
 765 static int torture_stutter(void *arg)
 766 {
 767         DEFINE_TORTURE_RANDOM(rand);
 768         int wtime;
 769
 770         VERBOSE_TOROUT_STRING("torture_stutter task started");
 771         do {
 772                 if (!torture_must_stop() && stutter > 1) {
 773                         wtime = stutter;
 774                         if (stutter > 2) {
 775                                 WRITE_ONCE(stutter_pause_test, 1);
 776                                 wtime = stutter - 3;
 777                                 torture_hrtimeout_jiffies(wtime, &rand);
 778                                 wtime = 2;
 779                         }
 780                         WRITE_ONCE(stutter_pause_test, 2);
 781                         torture_hrtimeout_jiffies(wtime, NULL);
 782                 }
 783                 WRITE_ONCE(stutter_pause_test, 0);
 784                 if (!torture_must_stop())
 785                         torture_hrtimeout_jiffies(stutter_gap, NULL);
 786                 torture_shutdown_absorb("torture_stutter");
 787         } while (!torture_must_stop());
 788         torture_kthread_stopping("torture_stutter");
 789         return 0;
 790 }
 791
 792 /*
 793  * Initialize and kick off the torture_stutter kthread.
 794  */
 795 int torture_stutter_init(const int s, const int sgap)
 796 {
 797         stutter = s;
 798         stutter_gap = sgap;
 799         return torture_create_kthread(torture_stutter, NULL, stutter_task);
 800 }
 801 EXPORT_SYMBOL_GPL(torture_stutter_init);
 802
 803 /*
 804  * Cleanup after the torture_stutter kthread.
 805  */
 806 static void torture_stutter_cleanup(void)
 807 {
 808         if (!stutter_task)
 809                 return;
 810         VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
 811         kthread_stop(stutter_task);
 812         stutter_task = NULL;
 813 }
 814
 815 /*
 816  * Initialize torture module.  Please note that this is -not- invoked via
 817  * the usual module_init() mechanism, but rather by an explicit call from
 818  * the client torture module.  This call must be paired with a later
 819  * torture_init_end().
 820  *
 821  * The runnable parameter points to a flag that controls whether or not
 822  * the test is currently runnable.  If there is no such flag, pass in NULL.
 823  */
 824 bool torture_init_begin(char *ttype, int v)
 825 {
 826         mutex_lock(&fullstop_mutex);
 827         if (torture_type != NULL) {
 828                 pr_alert("%s: Refusing %s init: %s running.\n",
 829                           __func__, ttype, torture_type);
 830                 pr_alert("%s: One torture test at a time!\n", __func__);
 831                 mutex_unlock(&fullstop_mutex);
 832                 return false;
 833         }
 834         torture_type = ttype;
 835         verbose = v;
 836         fullstop = FULLSTOP_DONTSTOP;
 837         return true;
 838 }
 839 EXPORT_SYMBOL_GPL(torture_init_begin);
 840
 841 /*
 842  * Tell the torture module that initialization is complete.
 843  */
 844 void torture_init_end(void)
 845 {
 846         mutex_unlock(&fullstop_mutex);
 847         register_reboot_notifier(&torture_shutdown_nb);
 848 }
 849 EXPORT_SYMBOL_GPL(torture_init_end);
 850
 851 /*
 852  * Clean up torture module.  Please note that this is -not- invoked via
 853  * the usual module_exit() mechanism, but rather by an explicit call from
 854  * the client torture module.  Returns true if a race with system shutdown
 855  * is detected, otherwise, all kthreads started by functions in this file
 856  * will be shut down.
 857  *
 858  * This must be called before the caller starts shutting down its own
 859  * kthreads.
 860  *
 861  * Both torture_cleanup_begin() and torture_cleanup_end() must be paired,
 862  * in order to correctly perform the cleanup. They are separated because
 863  * threads can still need to reference the torture_type type, thus nullify
 864  * only after completing all other relevant calls.
 865  */
 866 bool torture_cleanup_begin(void)
 867 {
 868         mutex_lock(&fullstop_mutex);
 869         if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 870                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 871                 mutex_unlock(&fullstop_mutex);
 872                 schedule_timeout_uninterruptible(10);
 873                 return true;
 874         }
 875         WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
 876         mutex_unlock(&fullstop_mutex);
 877         torture_shutdown_cleanup();
 878         torture_shuffle_cleanup();
 879         torture_stutter_cleanup();
 880         torture_onoff_cleanup();
 881         return false;
 882 }
 883 EXPORT_SYMBOL_GPL(torture_cleanup_begin);
 884
 885 void torture_cleanup_end(void)
 886 {
 887         mutex_lock(&fullstop_mutex);
 888         torture_type = NULL;
 889         mutex_unlock(&fullstop_mutex);
 890 }
 891 EXPORT_SYMBOL_GPL(torture_cleanup_end);
 892
 893 /*
 894  * Is it time for the current torture test to stop?
 895  */
 896 bool torture_must_stop(void)
 897 {
 898         return torture_must_stop_irq() || kthread_should_stop();
 899 }
 900 EXPORT_SYMBOL_GPL(torture_must_stop);
 901
 902 /*
 903  * Is it time for the current torture test to stop?  This is the irq-safe
 904  * version, hence no check for kthread_should_stop().
 905  */
 906 bool torture_must_stop_irq(void)
 907 {
 908         return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
 909 }
 910 EXPORT_SYMBOL_GPL(torture_must_stop_irq);
 911
 912 /*
 913  * Each kthread must wait for kthread_should_stop() before returning from
 914  * its top-level function, otherwise segfaults ensue.  This function
 915  * prints a "stopping" message and waits for kthread_should_stop(), and
 916  * should be called from all torture kthreads immediately prior to
 917  * returning.
 918  */
 919 void torture_kthread_stopping(char *title)
 920 {
 921         char buf[128];
 922
 923         snprintf(buf, sizeof(buf), "%s is stopping", title);
 924         VERBOSE_TOROUT_STRING(buf);
 925         while (!kthread_should_stop()) {
 926                 torture_shutdown_absorb(title);
 927                 schedule_timeout_uninterruptible(HZ / 20);
 928         }
 929 }
 930 EXPORT_SYMBOL_GPL(torture_kthread_stopping);
 931
 932 /*
 933  * Create a generic torture kthread that is immediately runnable.  If you
 934  * need the kthread to be stopped so that you can do something to it before
 935  * it starts, you will need to open-code your own.
 936  */
 937 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
 938                             char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp))
 939 {
 940         int ret = 0;
 941
 942         VERBOSE_TOROUT_STRING(m);
 943         *tp = kthread_create(fn, arg, "%s", s);
 944         if (IS_ERR(*tp)) {
 945                 ret = PTR_ERR(*tp);
 946                 TOROUT_ERRSTRING(f);
 947                 *tp = NULL;
 948                 return ret;
 949         }
 950
 951         if (cbf)
 952                 cbf(*tp);
 953
 954         wake_up_process(*tp);  // Process is sleeping, so ordering provided.
 955         torture_shuffle_task_register(*tp);
 956         return ret;
 957 }
 958 EXPORT_SYMBOL_GPL(_torture_create_kthread);
 959
 960 /*
 961  * Stop a generic kthread, emitting a message.
 962  */
 963 void _torture_stop_kthread(char *m, struct task_struct **tp)
 964 {
 965         if (*tp == NULL)
 966                 return;
 967         VERBOSE_TOROUT_STRING(m);
 968         kthread_stop(*tp);
 969         *tp = NULL;
 970 }
 971 EXPORT_SYMBOL_GPL(_torture_stop_kthread);