Merge tag 'landlock-6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/mic...
[platform/kernel/linux-starfive.git] / kernel / torture.c
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Common functions for in-kernel torture tests.
4  *
5  * Copyright (C) IBM Corporation, 2014
6  *
7  * Author: Paul E. McKenney <paulmck@linux.ibm.com>
8  *      Based on kernel/rcu/torture.c.
9  */
10
11 #define pr_fmt(fmt) fmt
12
13 #include <linux/types.h>
14 #include <linux/kernel.h>
15 #include <linux/init.h>
16 #include <linux/module.h>
17 #include <linux/kthread.h>
18 #include <linux/err.h>
19 #include <linux/spinlock.h>
20 #include <linux/smp.h>
21 #include <linux/interrupt.h>
22 #include <linux/sched.h>
23 #include <linux/sched/clock.h>
24 #include <linux/atomic.h>
25 #include <linux/bitops.h>
26 #include <linux/completion.h>
27 #include <linux/moduleparam.h>
28 #include <linux/percpu.h>
29 #include <linux/notifier.h>
30 #include <linux/reboot.h>
31 #include <linux/freezer.h>
32 #include <linux/cpu.h>
33 #include <linux/delay.h>
34 #include <linux/stat.h>
35 #include <linux/slab.h>
36 #include <linux/trace_clock.h>
37 #include <linux/ktime.h>
38 #include <asm/byteorder.h>
39 #include <linux/torture.h>
40 #include <linux/sched/rt.h>
41 #include "rcu/rcu.h"
42
43 MODULE_LICENSE("GPL");
44 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
45
46 static bool disable_onoff_at_boot;
47 module_param(disable_onoff_at_boot, bool, 0444);
48
49 static bool ftrace_dump_at_shutdown;
50 module_param(ftrace_dump_at_shutdown, bool, 0444);
51
52 static int verbose_sleep_frequency;
53 module_param(verbose_sleep_frequency, int, 0444);
54
55 static int verbose_sleep_duration = 1;
56 module_param(verbose_sleep_duration, int, 0444);
57
58 static int random_shuffle;
59 module_param(random_shuffle, int, 0444);
60
61 static char *torture_type;
62 static int verbose;
63
64 /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
65 #define FULLSTOP_DONTSTOP 0     /* Normal operation. */
66 #define FULLSTOP_SHUTDOWN 1     /* System shutdown with torture running. */
67 #define FULLSTOP_RMMOD    2     /* Normal rmmod of torture. */
68 static int fullstop = FULLSTOP_RMMOD;
69 static DEFINE_MUTEX(fullstop_mutex);
70
71 static atomic_t verbose_sleep_counter;
72
73 /*
74  * Sleep if needed from VERBOSE_TOROUT*().
75  */
76 void verbose_torout_sleep(void)
77 {
78         if (verbose_sleep_frequency > 0 &&
79             verbose_sleep_duration > 0 &&
80             !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency))
81                 schedule_timeout_uninterruptible(verbose_sleep_duration);
82 }
83 EXPORT_SYMBOL_GPL(verbose_torout_sleep);
84
85 /*
86  * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit
87  * nanosecond random fuzz.  This function and its friends desynchronize
88  * testing from the timer wheel.
89  */
90 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp)
91 {
92         ktime_t hto = baset_ns;
93
94         if (trsp)
95                 hto += torture_random(trsp) % fuzzt_ns;
96         set_current_state(TASK_IDLE);
97         return schedule_hrtimeout(&hto, HRTIMER_MODE_REL);
98 }
99 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns);
100
101 /*
102  * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit
103  * nanosecond (not microsecond!) random fuzz.
104  */
105 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp)
106 {
107         ktime_t baset_ns = baset_us * NSEC_PER_USEC;
108
109         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
110 }
111 EXPORT_SYMBOL_GPL(torture_hrtimeout_us);
112
113 /*
114  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
115  * microsecond (not millisecond!) random fuzz.
116  */
117 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp)
118 {
119         ktime_t baset_ns = baset_ms * NSEC_PER_MSEC;
120         u32 fuzzt_ns;
121
122         if ((u32)~0U / NSEC_PER_USEC < fuzzt_us)
123                 fuzzt_ns = (u32)~0U;
124         else
125                 fuzzt_ns = fuzzt_us * NSEC_PER_USEC;
126         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
127 }
128 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms);
129
130 /*
131  * Schedule a high-resolution-timer sleep in jiffies, with an
132  * implied one-jiffy random fuzz.  This is intended to replace calls to
133  * schedule_timeout_interruptible() and friends.
134  */
135 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp)
136 {
137         ktime_t baset_ns = jiffies_to_nsecs(baset_j);
138
139         return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp);
140 }
141 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies);
142
143 /*
144  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
145  * millisecond (not second!) random fuzz.
146  */
147 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp)
148 {
149         ktime_t baset_ns = baset_s * NSEC_PER_SEC;
150         u32 fuzzt_ns;
151
152         if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms)
153                 fuzzt_ns = (u32)~0U;
154         else
155                 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC;
156         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
157 }
158 EXPORT_SYMBOL_GPL(torture_hrtimeout_s);
159
160 #ifdef CONFIG_HOTPLUG_CPU
161
162 /*
163  * Variables for online-offline handling.  Only present if CPU hotplug
164  * is enabled, otherwise does nothing.
165  */
166
167 static struct task_struct *onoff_task;
168 static long onoff_holdoff;
169 static long onoff_interval;
170 static torture_ofl_func *onoff_f;
171 static long n_offline_attempts;
172 static long n_offline_successes;
173 static unsigned long sum_offline;
174 static int min_offline = -1;
175 static int max_offline;
176 static long n_online_attempts;
177 static long n_online_successes;
178 static unsigned long sum_online;
179 static int min_online = -1;
180 static int max_online;
181
182 static int torture_online_cpus = NR_CPUS;
183
184 /*
185  * Some torture testing leverages confusion as to the number of online
186  * CPUs.  This function returns the torture-testing view of this number,
187  * which allows torture tests to load-balance appropriately.
188  */
189 int torture_num_online_cpus(void)
190 {
191         return READ_ONCE(torture_online_cpus);
192 }
193 EXPORT_SYMBOL_GPL(torture_num_online_cpus);
194
195 /*
196  * Attempt to take a CPU offline.  Return false if the CPU is already
197  * offline or if it is not subject to CPU-hotplug operations.  The
198  * caller can detect other failures by looking at the statistics.
199  */
200 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
201                      unsigned long *sum_offl, int *min_offl, int *max_offl)
202 {
203         unsigned long delta;
204         int ret;
205         char *s;
206         unsigned long starttime;
207
208         if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
209                 return false;
210         if (num_online_cpus() <= 1)
211                 return false;  /* Can't offline the last CPU. */
212
213         if (verbose > 1)
214                 pr_alert("%s" TORTURE_FLAG
215                          "torture_onoff task: offlining %d\n",
216                          torture_type, cpu);
217         starttime = jiffies;
218         (*n_offl_attempts)++;
219         ret = remove_cpu(cpu);
220         if (ret) {
221                 s = "";
222                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
223                         // PCI probe frequently disables hotplug during boot.
224                         (*n_offl_attempts)--;
225                         s = " (-EBUSY forgiven during boot)";
226                 }
227                 if (verbose)
228                         pr_alert("%s" TORTURE_FLAG
229                                  "torture_onoff task: offline %d failed%s: errno %d\n",
230                                  torture_type, cpu, s, ret);
231         } else {
232                 if (verbose > 1)
233                         pr_alert("%s" TORTURE_FLAG
234                                  "torture_onoff task: offlined %d\n",
235                                  torture_type, cpu);
236                 if (onoff_f)
237                         onoff_f();
238                 (*n_offl_successes)++;
239                 delta = jiffies - starttime;
240                 *sum_offl += delta;
241                 if (*min_offl < 0) {
242                         *min_offl = delta;
243                         *max_offl = delta;
244                 }
245                 if (*min_offl > delta)
246                         *min_offl = delta;
247                 if (*max_offl < delta)
248                         *max_offl = delta;
249                 WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1);
250                 WARN_ON_ONCE(torture_online_cpus <= 0);
251         }
252
253         return true;
254 }
255 EXPORT_SYMBOL_GPL(torture_offline);
256
257 /*
258  * Attempt to bring a CPU online.  Return false if the CPU is already
259  * online or if it is not subject to CPU-hotplug operations.  The
260  * caller can detect other failures by looking at the statistics.
261  */
262 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
263                     unsigned long *sum_onl, int *min_onl, int *max_onl)
264 {
265         unsigned long delta;
266         int ret;
267         char *s;
268         unsigned long starttime;
269
270         if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
271                 return false;
272
273         if (verbose > 1)
274                 pr_alert("%s" TORTURE_FLAG
275                          "torture_onoff task: onlining %d\n",
276                          torture_type, cpu);
277         starttime = jiffies;
278         (*n_onl_attempts)++;
279         ret = add_cpu(cpu);
280         if (ret) {
281                 s = "";
282                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
283                         // PCI probe frequently disables hotplug during boot.
284                         (*n_onl_attempts)--;
285                         s = " (-EBUSY forgiven during boot)";
286                 }
287                 if (verbose)
288                         pr_alert("%s" TORTURE_FLAG
289                                  "torture_onoff task: online %d failed%s: errno %d\n",
290                                  torture_type, cpu, s, ret);
291         } else {
292                 if (verbose > 1)
293                         pr_alert("%s" TORTURE_FLAG
294                                  "torture_onoff task: onlined %d\n",
295                                  torture_type, cpu);
296                 (*n_onl_successes)++;
297                 delta = jiffies - starttime;
298                 *sum_onl += delta;
299                 if (*min_onl < 0) {
300                         *min_onl = delta;
301                         *max_onl = delta;
302                 }
303                 if (*min_onl > delta)
304                         *min_onl = delta;
305                 if (*max_onl < delta)
306                         *max_onl = delta;
307                 WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1);
308         }
309
310         return true;
311 }
312 EXPORT_SYMBOL_GPL(torture_online);
313
314 /*
315  * Get everything online at the beginning and ends of tests.
316  */
317 static void torture_online_all(char *phase)
318 {
319         int cpu;
320         int ret;
321
322         for_each_possible_cpu(cpu) {
323                 if (cpu_online(cpu))
324                         continue;
325                 ret = add_cpu(cpu);
326                 if (ret && verbose) {
327                         pr_alert("%s" TORTURE_FLAG
328                                  "%s: %s online %d: errno %d\n",
329                                  __func__, phase, torture_type, cpu, ret);
330                 }
331         }
332 }
333
334 /*
335  * Execute random CPU-hotplug operations at the interval specified
336  * by the onoff_interval.
337  */
338 static int
339 torture_onoff(void *arg)
340 {
341         int cpu;
342         int maxcpu = -1;
343         DEFINE_TORTURE_RANDOM(rand);
344
345         VERBOSE_TOROUT_STRING("torture_onoff task started");
346         for_each_online_cpu(cpu)
347                 maxcpu = cpu;
348         WARN_ON(maxcpu < 0);
349         torture_online_all("Initial");
350         if (maxcpu == 0) {
351                 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
352                 goto stop;
353         }
354
355         if (onoff_holdoff > 0) {
356                 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
357                 torture_hrtimeout_jiffies(onoff_holdoff, &rand);
358                 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
359         }
360         while (!torture_must_stop()) {
361                 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
362                         torture_hrtimeout_jiffies(HZ / 10, &rand);
363                         continue;
364                 }
365                 cpu = torture_random(&rand) % (maxcpu + 1);
366                 if (!torture_offline(cpu,
367                                      &n_offline_attempts, &n_offline_successes,
368                                      &sum_offline, &min_offline, &max_offline))
369                         torture_online(cpu,
370                                        &n_online_attempts, &n_online_successes,
371                                        &sum_online, &min_online, &max_online);
372                 torture_hrtimeout_jiffies(onoff_interval, &rand);
373         }
374
375 stop:
376         torture_kthread_stopping("torture_onoff");
377         torture_online_all("Final");
378         return 0;
379 }
380
381 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
382
383 /*
384  * Initiate online-offline handling.
385  */
386 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f)
387 {
388 #ifdef CONFIG_HOTPLUG_CPU
389         onoff_holdoff = ooholdoff;
390         onoff_interval = oointerval;
391         onoff_f = f;
392         if (onoff_interval <= 0)
393                 return 0;
394         return torture_create_kthread(torture_onoff, NULL, onoff_task);
395 #else /* #ifdef CONFIG_HOTPLUG_CPU */
396         return 0;
397 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
398 }
399 EXPORT_SYMBOL_GPL(torture_onoff_init);
400
401 /*
402  * Clean up after online/offline testing.
403  */
404 static void torture_onoff_cleanup(void)
405 {
406 #ifdef CONFIG_HOTPLUG_CPU
407         if (onoff_task == NULL)
408                 return;
409         VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
410         kthread_stop(onoff_task);
411         onoff_task = NULL;
412 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
413 }
414
415 /*
416  * Print online/offline testing statistics.
417  */
418 void torture_onoff_stats(void)
419 {
420 #ifdef CONFIG_HOTPLUG_CPU
421         pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
422                 n_online_successes, n_online_attempts,
423                 n_offline_successes, n_offline_attempts,
424                 min_online, max_online,
425                 min_offline, max_offline,
426                 sum_online, sum_offline, HZ);
427 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
428 }
429 EXPORT_SYMBOL_GPL(torture_onoff_stats);
430
431 /*
432  * Were all the online/offline operations successful?
433  */
434 bool torture_onoff_failures(void)
435 {
436 #ifdef CONFIG_HOTPLUG_CPU
437         return n_online_successes != n_online_attempts ||
438                n_offline_successes != n_offline_attempts;
439 #else /* #ifdef CONFIG_HOTPLUG_CPU */
440         return false;
441 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
442 }
443 EXPORT_SYMBOL_GPL(torture_onoff_failures);
444
445 #define TORTURE_RANDOM_MULT     39916801  /* prime */
446 #define TORTURE_RANDOM_ADD      479001701 /* prime */
447 #define TORTURE_RANDOM_REFRESH  10000
448
449 /*
450  * Crude but fast random-number generator.  Uses a linear congruential
451  * generator, with occasional help from cpu_clock().
452  */
453 unsigned long
454 torture_random(struct torture_random_state *trsp)
455 {
456         if (--trsp->trs_count < 0) {
457                 trsp->trs_state += (unsigned long)local_clock() + raw_smp_processor_id();
458                 trsp->trs_count = TORTURE_RANDOM_REFRESH;
459         }
460         trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
461                 TORTURE_RANDOM_ADD;
462         return swahw32(trsp->trs_state);
463 }
464 EXPORT_SYMBOL_GPL(torture_random);
465
466 /*
467  * Variables for shuffling.  The idea is to ensure that each CPU stays
468  * idle for an extended period to test interactions with dyntick idle,
469  * as well as interactions with any per-CPU variables.
470  */
471 struct shuffle_task {
472         struct list_head st_l;
473         struct task_struct *st_t;
474 };
475
476 static long shuffle_interval;   /* In jiffies. */
477 static struct task_struct *shuffler_task;
478 static cpumask_var_t shuffle_tmp_mask;
479 static int shuffle_idle_cpu;    /* Force all torture tasks off this CPU */
480 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
481 static DEFINE_MUTEX(shuffle_task_mutex);
482
483 /*
484  * Register a task to be shuffled.  If there is no memory, just splat
485  * and don't bother registering.
486  */
487 void torture_shuffle_task_register(struct task_struct *tp)
488 {
489         struct shuffle_task *stp;
490
491         if (WARN_ON_ONCE(tp == NULL))
492                 return;
493         stp = kmalloc(sizeof(*stp), GFP_KERNEL);
494         if (WARN_ON_ONCE(stp == NULL))
495                 return;
496         stp->st_t = tp;
497         mutex_lock(&shuffle_task_mutex);
498         list_add(&stp->st_l, &shuffle_task_list);
499         mutex_unlock(&shuffle_task_mutex);
500 }
501 EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
502
503 /*
504  * Unregister all tasks, for example, at the end of the torture run.
505  */
506 static void torture_shuffle_task_unregister_all(void)
507 {
508         struct shuffle_task *stp;
509         struct shuffle_task *p;
510
511         mutex_lock(&shuffle_task_mutex);
512         list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
513                 list_del(&stp->st_l);
514                 kfree(stp);
515         }
516         mutex_unlock(&shuffle_task_mutex);
517 }
518
519 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle.
520  * A special case is when shuffle_idle_cpu = -1, in which case we allow
521  * the tasks to run on all CPUs.
522  */
523 static void torture_shuffle_tasks(void)
524 {
525         DEFINE_TORTURE_RANDOM(rand);
526         struct shuffle_task *stp;
527
528         cpumask_setall(shuffle_tmp_mask);
529         cpus_read_lock();
530
531         /* No point in shuffling if there is only one online CPU (ex: UP) */
532         if (num_online_cpus() == 1) {
533                 cpus_read_unlock();
534                 return;
535         }
536
537         /* Advance to the next CPU.  Upon overflow, don't idle any CPUs. */
538         shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
539         if (shuffle_idle_cpu >= nr_cpu_ids)
540                 shuffle_idle_cpu = -1;
541         else
542                 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
543
544         mutex_lock(&shuffle_task_mutex);
545         list_for_each_entry(stp, &shuffle_task_list, st_l) {
546                 if (!random_shuffle || torture_random(&rand) & 0x1)
547                         set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
548         }
549         mutex_unlock(&shuffle_task_mutex);
550
551         cpus_read_unlock();
552 }
553
554 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
555  * system to become idle at a time and cut off its timer ticks. This is meant
556  * to test the support for such tickless idle CPU in RCU.
557  */
558 static int torture_shuffle(void *arg)
559 {
560         DEFINE_TORTURE_RANDOM(rand);
561
562         VERBOSE_TOROUT_STRING("torture_shuffle task started");
563         do {
564                 torture_hrtimeout_jiffies(shuffle_interval, &rand);
565                 torture_shuffle_tasks();
566                 torture_shutdown_absorb("torture_shuffle");
567         } while (!torture_must_stop());
568         torture_kthread_stopping("torture_shuffle");
569         return 0;
570 }
571
572 /*
573  * Start the shuffler, with shuffint in jiffies.
574  */
575 int torture_shuffle_init(long shuffint)
576 {
577         shuffle_interval = shuffint;
578
579         shuffle_idle_cpu = -1;
580
581         if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
582                 TOROUT_ERRSTRING("Failed to alloc mask");
583                 return -ENOMEM;
584         }
585
586         /* Create the shuffler thread */
587         return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
588 }
589 EXPORT_SYMBOL_GPL(torture_shuffle_init);
590
591 /*
592  * Stop the shuffling.
593  */
594 static void torture_shuffle_cleanup(void)
595 {
596         torture_shuffle_task_unregister_all();
597         if (shuffler_task) {
598                 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
599                 kthread_stop(shuffler_task);
600                 free_cpumask_var(shuffle_tmp_mask);
601         }
602         shuffler_task = NULL;
603 }
604
605 /*
606  * Variables for auto-shutdown.  This allows "lights out" torture runs
607  * to be fully scripted.
608  */
609 static struct task_struct *shutdown_task;
610 static ktime_t shutdown_time;           /* time to system shutdown. */
611 static void (*torture_shutdown_hook)(void);
612
613 /*
614  * Absorb kthreads into a kernel function that won't return, so that
615  * they won't ever access module text or data again.
616  */
617 void torture_shutdown_absorb(const char *title)
618 {
619         while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
620                 pr_notice("torture thread %s parking due to system shutdown\n",
621                           title);
622                 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
623         }
624 }
625 EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
626
627 /*
628  * Cause the torture test to shutdown the system after the test has
629  * run for the time specified by the shutdown_secs parameter.
630  */
631 static int torture_shutdown(void *arg)
632 {
633         ktime_t ktime_snap;
634
635         VERBOSE_TOROUT_STRING("torture_shutdown task started");
636         ktime_snap = ktime_get();
637         while (ktime_before(ktime_snap, shutdown_time) &&
638                !torture_must_stop()) {
639                 if (verbose)
640                         pr_alert("%s" TORTURE_FLAG
641                                  "torture_shutdown task: %llu ms remaining\n",
642                                  torture_type,
643                                  ktime_ms_delta(shutdown_time, ktime_snap));
644                 set_current_state(TASK_INTERRUPTIBLE);
645                 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
646                 ktime_snap = ktime_get();
647         }
648         if (torture_must_stop()) {
649                 torture_kthread_stopping("torture_shutdown");
650                 return 0;
651         }
652
653         /* OK, shut down the system. */
654
655         VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
656         shutdown_task = NULL;   /* Avoid self-kill deadlock. */
657         if (torture_shutdown_hook)
658                 torture_shutdown_hook();
659         else
660                 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
661         if (ftrace_dump_at_shutdown)
662                 rcu_ftrace_dump(DUMP_ALL);
663         kernel_power_off();     /* Shut down the system. */
664         return 0;
665 }
666
667 /*
668  * Start up the shutdown task.
669  */
670 int torture_shutdown_init(int ssecs, void (*cleanup)(void))
671 {
672         torture_shutdown_hook = cleanup;
673         if (ssecs > 0) {
674                 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
675                 return torture_create_kthread(torture_shutdown, NULL,
676                                              shutdown_task);
677         }
678         return 0;
679 }
680 EXPORT_SYMBOL_GPL(torture_shutdown_init);
681
682 /*
683  * Detect and respond to a system shutdown.
684  */
685 static int torture_shutdown_notify(struct notifier_block *unused1,
686                                    unsigned long unused2, void *unused3)
687 {
688         mutex_lock(&fullstop_mutex);
689         if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
690                 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
691                 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
692         } else {
693                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
694         }
695         mutex_unlock(&fullstop_mutex);
696         return NOTIFY_DONE;
697 }
698
699 static struct notifier_block torture_shutdown_nb = {
700         .notifier_call = torture_shutdown_notify,
701 };
702
703 /*
704  * Shut down the shutdown task.  Say what???  Heh!  This can happen if
705  * the torture module gets an rmmod before the shutdown time arrives.  ;-)
706  */
707 static void torture_shutdown_cleanup(void)
708 {
709         unregister_reboot_notifier(&torture_shutdown_nb);
710         if (shutdown_task != NULL) {
711                 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
712                 kthread_stop(shutdown_task);
713         }
714         shutdown_task = NULL;
715 }
716
717 /*
718  * Variables for stuttering, which means to periodically pause and
719  * restart testing in order to catch bugs that appear when load is
720  * suddenly applied to or removed from the system.
721  */
722 static struct task_struct *stutter_task;
723 static int stutter_pause_test;
724 static int stutter;
725 static int stutter_gap;
726
727 /*
728  * Block until the stutter interval ends.  This must be called periodically
729  * by all running kthreads that need to be subject to stuttering.
730  */
731 bool stutter_wait(const char *title)
732 {
733         unsigned int i = 0;
734         bool ret = false;
735         int spt;
736
737         cond_resched_tasks_rcu_qs();
738         spt = READ_ONCE(stutter_pause_test);
739         for (; spt; spt = READ_ONCE(stutter_pause_test)) {
740                 if (!ret && !rt_task(current)) {
741                         sched_set_normal(current, MAX_NICE);
742                         ret = true;
743                 }
744                 if (spt == 1) {
745                         torture_hrtimeout_jiffies(1, NULL);
746                 } else if (spt == 2) {
747                         while (READ_ONCE(stutter_pause_test)) {
748                                 if (!(i++ & 0xffff))
749                                         torture_hrtimeout_us(10, 0, NULL);
750                                 cond_resched();
751                         }
752                 } else {
753                         torture_hrtimeout_jiffies(round_jiffies_relative(HZ), NULL);
754                 }
755                 torture_shutdown_absorb(title);
756         }
757         return ret;
758 }
759 EXPORT_SYMBOL_GPL(stutter_wait);
760
761 /*
762  * Cause the torture test to "stutter", starting and stopping all
763  * threads periodically.
764  */
765 static int torture_stutter(void *arg)
766 {
767         DEFINE_TORTURE_RANDOM(rand);
768         int wtime;
769
770         VERBOSE_TOROUT_STRING("torture_stutter task started");
771         do {
772                 if (!torture_must_stop() && stutter > 1) {
773                         wtime = stutter;
774                         if (stutter > 2) {
775                                 WRITE_ONCE(stutter_pause_test, 1);
776                                 wtime = stutter - 3;
777                                 torture_hrtimeout_jiffies(wtime, &rand);
778                                 wtime = 2;
779                         }
780                         WRITE_ONCE(stutter_pause_test, 2);
781                         torture_hrtimeout_jiffies(wtime, NULL);
782                 }
783                 WRITE_ONCE(stutter_pause_test, 0);
784                 if (!torture_must_stop())
785                         torture_hrtimeout_jiffies(stutter_gap, NULL);
786                 torture_shutdown_absorb("torture_stutter");
787         } while (!torture_must_stop());
788         torture_kthread_stopping("torture_stutter");
789         return 0;
790 }
791
792 /*
793  * Initialize and kick off the torture_stutter kthread.
794  */
795 int torture_stutter_init(const int s, const int sgap)
796 {
797         stutter = s;
798         stutter_gap = sgap;
799         return torture_create_kthread(torture_stutter, NULL, stutter_task);
800 }
801 EXPORT_SYMBOL_GPL(torture_stutter_init);
802
803 /*
804  * Cleanup after the torture_stutter kthread.
805  */
806 static void torture_stutter_cleanup(void)
807 {
808         if (!stutter_task)
809                 return;
810         VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
811         kthread_stop(stutter_task);
812         stutter_task = NULL;
813 }
814
815 /*
816  * Initialize torture module.  Please note that this is -not- invoked via
817  * the usual module_init() mechanism, but rather by an explicit call from
818  * the client torture module.  This call must be paired with a later
819  * torture_init_end().
820  *
821  * The runnable parameter points to a flag that controls whether or not
822  * the test is currently runnable.  If there is no such flag, pass in NULL.
823  */
824 bool torture_init_begin(char *ttype, int v)
825 {
826         mutex_lock(&fullstop_mutex);
827         if (torture_type != NULL) {
828                 pr_alert("%s: Refusing %s init: %s running.\n",
829                           __func__, ttype, torture_type);
830                 pr_alert("%s: One torture test at a time!\n", __func__);
831                 mutex_unlock(&fullstop_mutex);
832                 return false;
833         }
834         torture_type = ttype;
835         verbose = v;
836         fullstop = FULLSTOP_DONTSTOP;
837         return true;
838 }
839 EXPORT_SYMBOL_GPL(torture_init_begin);
840
841 /*
842  * Tell the torture module that initialization is complete.
843  */
844 void torture_init_end(void)
845 {
846         mutex_unlock(&fullstop_mutex);
847         register_reboot_notifier(&torture_shutdown_nb);
848 }
849 EXPORT_SYMBOL_GPL(torture_init_end);
850
851 /*
852  * Clean up torture module.  Please note that this is -not- invoked via
853  * the usual module_exit() mechanism, but rather by an explicit call from
854  * the client torture module.  Returns true if a race with system shutdown
855  * is detected, otherwise, all kthreads started by functions in this file
856  * will be shut down.
857  *
858  * This must be called before the caller starts shutting down its own
859  * kthreads.
860  *
861  * Both torture_cleanup_begin() and torture_cleanup_end() must be paired,
862  * in order to correctly perform the cleanup. They are separated because
863  * threads can still need to reference the torture_type type, thus nullify
864  * only after completing all other relevant calls.
865  */
866 bool torture_cleanup_begin(void)
867 {
868         mutex_lock(&fullstop_mutex);
869         if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
870                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
871                 mutex_unlock(&fullstop_mutex);
872                 schedule_timeout_uninterruptible(10);
873                 return true;
874         }
875         WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
876         mutex_unlock(&fullstop_mutex);
877         torture_shutdown_cleanup();
878         torture_shuffle_cleanup();
879         torture_stutter_cleanup();
880         torture_onoff_cleanup();
881         return false;
882 }
883 EXPORT_SYMBOL_GPL(torture_cleanup_begin);
884
885 void torture_cleanup_end(void)
886 {
887         mutex_lock(&fullstop_mutex);
888         torture_type = NULL;
889         mutex_unlock(&fullstop_mutex);
890 }
891 EXPORT_SYMBOL_GPL(torture_cleanup_end);
892
893 /*
894  * Is it time for the current torture test to stop?
895  */
896 bool torture_must_stop(void)
897 {
898         return torture_must_stop_irq() || kthread_should_stop();
899 }
900 EXPORT_SYMBOL_GPL(torture_must_stop);
901
902 /*
903  * Is it time for the current torture test to stop?  This is the irq-safe
904  * version, hence no check for kthread_should_stop().
905  */
906 bool torture_must_stop_irq(void)
907 {
908         return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
909 }
910 EXPORT_SYMBOL_GPL(torture_must_stop_irq);
911
912 /*
913  * Each kthread must wait for kthread_should_stop() before returning from
914  * its top-level function, otherwise segfaults ensue.  This function
915  * prints a "stopping" message and waits for kthread_should_stop(), and
916  * should be called from all torture kthreads immediately prior to
917  * returning.
918  */
919 void torture_kthread_stopping(char *title)
920 {
921         char buf[128];
922
923         snprintf(buf, sizeof(buf), "%s is stopping", title);
924         VERBOSE_TOROUT_STRING(buf);
925         while (!kthread_should_stop()) {
926                 torture_shutdown_absorb(title);
927                 schedule_timeout_uninterruptible(HZ / 20);
928         }
929 }
930 EXPORT_SYMBOL_GPL(torture_kthread_stopping);
931
932 /*
933  * Create a generic torture kthread that is immediately runnable.  If you
934  * need the kthread to be stopped so that you can do something to it before
935  * it starts, you will need to open-code your own.
936  */
937 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
938                             char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp))
939 {
940         int ret = 0;
941
942         VERBOSE_TOROUT_STRING(m);
943         *tp = kthread_create(fn, arg, "%s", s);
944         if (IS_ERR(*tp)) {
945                 ret = PTR_ERR(*tp);
946                 TOROUT_ERRSTRING(f);
947                 *tp = NULL;
948                 return ret;
949         }
950
951         if (cbf)
952                 cbf(*tp);
953
954         wake_up_process(*tp);  // Process is sleeping, so ordering provided.
955         torture_shuffle_task_register(*tp);
956         return ret;
957 }
958 EXPORT_SYMBOL_GPL(_torture_create_kthread);
959
960 /*
961  * Stop a generic kthread, emitting a message.
962  */
963 void _torture_stop_kthread(char *m, struct task_struct **tp)
964 {
965         if (*tp == NULL)
966                 return;
967         VERBOSE_TOROUT_STRING(m);
968         kthread_stop(*tp);
969         *tp = NULL;
970 }
971 EXPORT_SYMBOL_GPL(_torture_stop_kthread);