kernel/locking/locktorture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Module-based torture test facility for locking
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *          Davidlohr Bueso <dave@stgolabs.net>
   9  *      Based on kernel/rcu/torture.c.
  10  */
  11
  12 #define pr_fmt(fmt) fmt
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/kthread.h>
  17 #include <linux/sched/rt.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rwsem.h>
  21 #include <linux/smp.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/sched.h>
  24 #include <uapi/linux/sched/types.h>
  25 #include <linux/rtmutex.h>
  26 #include <linux/atomic.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/delay.h>
  29 #include <linux/slab.h>
  30 #include <linux/torture.h>
  31 #include <linux/reboot.h>
  32
  33 MODULE_LICENSE("GPL");
  34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  35
  36 torture_param(int, nwriters_stress, -1, "Number of write-locking stress-test threads");
  37 torture_param(int, nreaders_stress, -1, "Number of read-locking stress-test threads");
  38 torture_param(int, long_hold, 100, "Do occasional long hold of lock (ms), 0=disable");
  39 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  40 torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable");
  41 torture_param(int, shuffle_interval, 3, "Number of jiffies between shuffles, 0=disable");
  42 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
  43 torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s");
  44 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
  45 torture_param(int, rt_boost, 2,
  46                    "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types.");
  47 torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens.");
  48 torture_param(int, writer_fifo, 0, "Run writers at sched_set_fifo() priority");
  49 torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
  50 torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)");
  51 /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */
  52 #define MAX_NESTED_LOCKS 8
  53
  54 static char *torture_type = IS_ENABLED(CONFIG_PREEMPT_RT) ? "raw_spin_lock" : "spin_lock";
  55 module_param(torture_type, charp, 0444);
  56 MODULE_PARM_DESC(torture_type,
  57                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
  58
  59 static struct task_struct *stats_task;
  60 static struct task_struct **writer_tasks;
  61 static struct task_struct **reader_tasks;
  62
  63 static bool lock_is_write_held;
  64 static atomic_t lock_is_read_held;
  65 static unsigned long last_lock_release;
  66
  67 struct lock_stress_stats {
  68         long n_lock_fail;
  69         long n_lock_acquired;
  70 };
  71
  72 /* Forward reference. */
  73 static void lock_torture_cleanup(void);
  74
  75 /*
  76  * Operations vector for selecting different types of tests.
  77  */
  78 struct lock_torture_ops {
  79         void (*init)(void);
  80         void (*exit)(void);
  81         int (*nested_lock)(int tid, u32 lockset);
  82         int (*writelock)(int tid);
  83         void (*write_delay)(struct torture_random_state *trsp);
  84         void (*task_boost)(struct torture_random_state *trsp);
  85         void (*writeunlock)(int tid);
  86         void (*nested_unlock)(int tid, u32 lockset);
  87         int (*readlock)(int tid);
  88         void (*read_delay)(struct torture_random_state *trsp);
  89         void (*readunlock)(int tid);
  90
  91         unsigned long flags; /* for irq spinlocks */
  92         const char *name;
  93 };
  94
  95 struct lock_torture_cxt {
  96         int nrealwriters_stress;
  97         int nrealreaders_stress;
  98         bool debug_lock;
  99         bool init_called;
 100         atomic_t n_lock_torture_errors;
 101         struct lock_torture_ops *cur_ops;
 102         struct lock_stress_stats *lwsa; /* writer statistics */
 103         struct lock_stress_stats *lrsa; /* reader statistics */
 104 };
 105 static struct lock_torture_cxt cxt = { 0, 0, false, false,
 106                                        ATOMIC_INIT(0),
 107                                        NULL, NULL};
 108 /*
 109  * Definitions for lock torture testing.
 110  */
 111
 112 static int torture_lock_busted_write_lock(int tid __maybe_unused)
 113 {
 114         return 0;  /* BUGGY, do not use in real life!!! */
 115 }
 116
 117 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
 118 {
 119         const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX;
 120
 121         /* We want a long delay occasionally to force massive contention.  */
 122         if (!(torture_random(trsp) %
 123               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 124                 mdelay(longdelay_ms);
 125         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 126                 torture_preempt_schedule();  /* Allow test to be preempted. */
 127 }
 128
 129 static void torture_lock_busted_write_unlock(int tid __maybe_unused)
 130 {
 131           /* BUGGY, do not use in real life!!! */
 132 }
 133
 134 static void __torture_rt_boost(struct torture_random_state *trsp)
 135 {
 136         const unsigned int factor = rt_boost_factor;
 137
 138         if (!rt_task(current)) {
 139                 /*
 140                  * Boost priority once every rt_boost_factor operations. When
 141                  * the task tries to take the lock, the rtmutex it will account
 142                  * for the new priority, and do any corresponding pi-dance.
 143                  */
 144                 if (trsp && !(torture_random(trsp) %
 145                               (cxt.nrealwriters_stress * factor))) {
 146                         sched_set_fifo(current);
 147                 } else /* common case, do nothing */
 148                         return;
 149         } else {
 150                 /*
 151                  * The task will remain boosted for another 10 * rt_boost_factor
 152                  * operations, then restored back to its original prio, and so
 153                  * forth.
 154                  *
 155                  * When @trsp is nil, we want to force-reset the task for
 156                  * stopping the kthread.
 157                  */
 158                 if (!trsp || !(torture_random(trsp) %
 159                                (cxt.nrealwriters_stress * factor * 2))) {
 160                         sched_set_normal(current, 0);
 161                 } else /* common case, do nothing */
 162                         return;
 163         }
 164 }
 165
 166 static void torture_rt_boost(struct torture_random_state *trsp)
 167 {
 168         if (rt_boost != 2)
 169                 return;
 170
 171         __torture_rt_boost(trsp);
 172 }
 173
 174 static struct lock_torture_ops lock_busted_ops = {
 175         .writelock      = torture_lock_busted_write_lock,
 176         .write_delay    = torture_lock_busted_write_delay,
 177         .task_boost     = torture_rt_boost,
 178         .writeunlock    = torture_lock_busted_write_unlock,
 179         .readlock       = NULL,
 180         .read_delay     = NULL,
 181         .readunlock     = NULL,
 182         .name           = "lock_busted"
 183 };
 184
 185 static DEFINE_SPINLOCK(torture_spinlock);
 186
 187 static int torture_spin_lock_write_lock(int tid __maybe_unused)
 188 __acquires(torture_spinlock)
 189 {
 190         spin_lock(&torture_spinlock);
 191         return 0;
 192 }
 193
 194 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
 195 {
 196         const unsigned long shortdelay_us = 2;
 197         const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX;
 198         unsigned long j;
 199
 200         /* We want a short delay mostly to emulate likely code, and
 201          * we want a long delay occasionally to force massive contention.
 202          */
 203         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * longdelay_ms))) {
 204                 j = jiffies;
 205                 mdelay(longdelay_ms);
 206                 pr_alert("%s: delay = %lu jiffies.\n", __func__, jiffies - j);
 207         }
 208         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 200 * shortdelay_us)))
 209                 udelay(shortdelay_us);
 210         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 211                 torture_preempt_schedule();  /* Allow test to be preempted. */
 212 }
 213
 214 static void torture_spin_lock_write_unlock(int tid __maybe_unused)
 215 __releases(torture_spinlock)
 216 {
 217         spin_unlock(&torture_spinlock);
 218 }
 219
 220 static struct lock_torture_ops spin_lock_ops = {
 221         .writelock      = torture_spin_lock_write_lock,
 222         .write_delay    = torture_spin_lock_write_delay,
 223         .task_boost     = torture_rt_boost,
 224         .writeunlock    = torture_spin_lock_write_unlock,
 225         .readlock       = NULL,
 226         .read_delay     = NULL,
 227         .readunlock     = NULL,
 228         .name           = "spin_lock"
 229 };
 230
 231 static int torture_spin_lock_write_lock_irq(int tid __maybe_unused)
 232 __acquires(torture_spinlock)
 233 {
 234         unsigned long flags;
 235
 236         spin_lock_irqsave(&torture_spinlock, flags);
 237         cxt.cur_ops->flags = flags;
 238         return 0;
 239 }
 240
 241 static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused)
 242 __releases(torture_spinlock)
 243 {
 244         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
 245 }
 246
 247 static struct lock_torture_ops spin_lock_irq_ops = {
 248         .writelock      = torture_spin_lock_write_lock_irq,
 249         .write_delay    = torture_spin_lock_write_delay,
 250         .task_boost     = torture_rt_boost,
 251         .writeunlock    = torture_lock_spin_write_unlock_irq,
 252         .readlock       = NULL,
 253         .read_delay     = NULL,
 254         .readunlock     = NULL,
 255         .name           = "spin_lock_irq"
 256 };
 257
 258 static DEFINE_RAW_SPINLOCK(torture_raw_spinlock);
 259
 260 static int torture_raw_spin_lock_write_lock(int tid __maybe_unused)
 261 __acquires(torture_raw_spinlock)
 262 {
 263         raw_spin_lock(&torture_raw_spinlock);
 264         return 0;
 265 }
 266
 267 static void torture_raw_spin_lock_write_unlock(int tid __maybe_unused)
 268 __releases(torture_raw_spinlock)
 269 {
 270         raw_spin_unlock(&torture_raw_spinlock);
 271 }
 272
 273 static struct lock_torture_ops raw_spin_lock_ops = {
 274         .writelock      = torture_raw_spin_lock_write_lock,
 275         .write_delay    = torture_spin_lock_write_delay,
 276         .task_boost     = torture_rt_boost,
 277         .writeunlock    = torture_raw_spin_lock_write_unlock,
 278         .readlock       = NULL,
 279         .read_delay     = NULL,
 280         .readunlock     = NULL,
 281         .name           = "raw_spin_lock"
 282 };
 283
 284 static int torture_raw_spin_lock_write_lock_irq(int tid __maybe_unused)
 285 __acquires(torture_raw_spinlock)
 286 {
 287         unsigned long flags;
 288
 289         raw_spin_lock_irqsave(&torture_raw_spinlock, flags);
 290         cxt.cur_ops->flags = flags;
 291         return 0;
 292 }
 293
 294 static void torture_raw_spin_lock_write_unlock_irq(int tid __maybe_unused)
 295 __releases(torture_raw_spinlock)
 296 {
 297         raw_spin_unlock_irqrestore(&torture_raw_spinlock, cxt.cur_ops->flags);
 298 }
 299
 300 static struct lock_torture_ops raw_spin_lock_irq_ops = {
 301         .writelock      = torture_raw_spin_lock_write_lock_irq,
 302         .write_delay    = torture_spin_lock_write_delay,
 303         .task_boost     = torture_rt_boost,
 304         .writeunlock    = torture_raw_spin_lock_write_unlock_irq,
 305         .readlock       = NULL,
 306         .read_delay     = NULL,
 307         .readunlock     = NULL,
 308         .name           = "raw_spin_lock_irq"
 309 };
 310
 311 static DEFINE_RWLOCK(torture_rwlock);
 312
 313 static int torture_rwlock_write_lock(int tid __maybe_unused)
 314 __acquires(torture_rwlock)
 315 {
 316         write_lock(&torture_rwlock);
 317         return 0;
 318 }
 319
 320 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
 321 {
 322         const unsigned long shortdelay_us = 2;
 323         const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX;
 324
 325         /* We want a short delay mostly to emulate likely code, and
 326          * we want a long delay occasionally to force massive contention.
 327          */
 328         if (!(torture_random(trsp) %
 329               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 330                 mdelay(longdelay_ms);
 331         else
 332                 udelay(shortdelay_us);
 333 }
 334
 335 static void torture_rwlock_write_unlock(int tid __maybe_unused)
 336 __releases(torture_rwlock)
 337 {
 338         write_unlock(&torture_rwlock);
 339 }
 340
 341 static int torture_rwlock_read_lock(int tid __maybe_unused)
 342 __acquires(torture_rwlock)
 343 {
 344         read_lock(&torture_rwlock);
 345         return 0;
 346 }
 347
 348 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
 349 {
 350         const unsigned long shortdelay_us = 10;
 351         const unsigned long longdelay_ms = 100;
 352
 353         /* We want a short delay mostly to emulate likely code, and
 354          * we want a long delay occasionally to force massive contention.
 355          */
 356         if (!(torture_random(trsp) %
 357               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 358                 mdelay(longdelay_ms);
 359         else
 360                 udelay(shortdelay_us);
 361 }
 362
 363 static void torture_rwlock_read_unlock(int tid __maybe_unused)
 364 __releases(torture_rwlock)
 365 {
 366         read_unlock(&torture_rwlock);
 367 }
 368
 369 static struct lock_torture_ops rw_lock_ops = {
 370         .writelock      = torture_rwlock_write_lock,
 371         .write_delay    = torture_rwlock_write_delay,
 372         .task_boost     = torture_rt_boost,
 373         .writeunlock    = torture_rwlock_write_unlock,
 374         .readlock       = torture_rwlock_read_lock,
 375         .read_delay     = torture_rwlock_read_delay,
 376         .readunlock     = torture_rwlock_read_unlock,
 377         .name           = "rw_lock"
 378 };
 379
 380 static int torture_rwlock_write_lock_irq(int tid __maybe_unused)
 381 __acquires(torture_rwlock)
 382 {
 383         unsigned long flags;
 384
 385         write_lock_irqsave(&torture_rwlock, flags);
 386         cxt.cur_ops->flags = flags;
 387         return 0;
 388 }
 389
 390 static void torture_rwlock_write_unlock_irq(int tid __maybe_unused)
 391 __releases(torture_rwlock)
 392 {
 393         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 394 }
 395
 396 static int torture_rwlock_read_lock_irq(int tid __maybe_unused)
 397 __acquires(torture_rwlock)
 398 {
 399         unsigned long flags;
 400
 401         read_lock_irqsave(&torture_rwlock, flags);
 402         cxt.cur_ops->flags = flags;
 403         return 0;
 404 }
 405
 406 static void torture_rwlock_read_unlock_irq(int tid __maybe_unused)
 407 __releases(torture_rwlock)
 408 {
 409         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 410 }
 411
 412 static struct lock_torture_ops rw_lock_irq_ops = {
 413         .writelock      = torture_rwlock_write_lock_irq,
 414         .write_delay    = torture_rwlock_write_delay,
 415         .task_boost     = torture_rt_boost,
 416         .writeunlock    = torture_rwlock_write_unlock_irq,
 417         .readlock       = torture_rwlock_read_lock_irq,
 418         .read_delay     = torture_rwlock_read_delay,
 419         .readunlock     = torture_rwlock_read_unlock_irq,
 420         .name           = "rw_lock_irq"
 421 };
 422
 423 static DEFINE_MUTEX(torture_mutex);
 424 static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS];
 425 static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS];
 426
 427 static void torture_mutex_init(void)
 428 {
 429         int i;
 430
 431         for (i = 0; i < MAX_NESTED_LOCKS; i++)
 432                 __mutex_init(&torture_nested_mutexes[i], __func__,
 433                              &nested_mutex_keys[i]);
 434 }
 435
 436 static int torture_mutex_nested_lock(int tid __maybe_unused,
 437                                      u32 lockset)
 438 {
 439         int i;
 440
 441         for (i = 0; i < nested_locks; i++)
 442                 if (lockset & (1 << i))
 443                         mutex_lock(&torture_nested_mutexes[i]);
 444         return 0;
 445 }
 446
 447 static int torture_mutex_lock(int tid __maybe_unused)
 448 __acquires(torture_mutex)
 449 {
 450         mutex_lock(&torture_mutex);
 451         return 0;
 452 }
 453
 454 static void torture_mutex_delay(struct torture_random_state *trsp)
 455 {
 456         const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX;
 457
 458         /* We want a long delay occasionally to force massive contention.  */
 459         if (!(torture_random(trsp) %
 460               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 461                 mdelay(longdelay_ms * 5);
 462         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 463                 torture_preempt_schedule();  /* Allow test to be preempted. */
 464 }
 465
 466 static void torture_mutex_unlock(int tid __maybe_unused)
 467 __releases(torture_mutex)
 468 {
 469         mutex_unlock(&torture_mutex);
 470 }
 471
 472 static void torture_mutex_nested_unlock(int tid __maybe_unused,
 473                                         u32 lockset)
 474 {
 475         int i;
 476
 477         for (i = nested_locks - 1; i >= 0; i--)
 478                 if (lockset & (1 << i))
 479                         mutex_unlock(&torture_nested_mutexes[i]);
 480 }
 481
 482 static struct lock_torture_ops mutex_lock_ops = {
 483         .init           = torture_mutex_init,
 484         .nested_lock    = torture_mutex_nested_lock,
 485         .writelock      = torture_mutex_lock,
 486         .write_delay    = torture_mutex_delay,
 487         .task_boost     = torture_rt_boost,
 488         .writeunlock    = torture_mutex_unlock,
 489         .nested_unlock  = torture_mutex_nested_unlock,
 490         .readlock       = NULL,
 491         .read_delay     = NULL,
 492         .readunlock     = NULL,
 493         .name           = "mutex_lock"
 494 };
 495
 496 #include <linux/ww_mutex.h>
 497 /*
 498  * The torture ww_mutexes should belong to the same lock class as
 499  * torture_ww_class to avoid lockdep problem. The ww_mutex_init()
 500  * function is called for initialization to ensure that.
 501  */
 502 static DEFINE_WD_CLASS(torture_ww_class);
 503 static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2;
 504 static struct ww_acquire_ctx *ww_acquire_ctxs;
 505
 506 static void torture_ww_mutex_init(void)
 507 {
 508         ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class);
 509         ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class);
 510         ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class);
 511
 512         ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress,
 513                                         sizeof(*ww_acquire_ctxs),
 514                                         GFP_KERNEL);
 515         if (!ww_acquire_ctxs)
 516                 VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory");
 517 }
 518
 519 static void torture_ww_mutex_exit(void)
 520 {
 521         kfree(ww_acquire_ctxs);
 522 }
 523
 524 static int torture_ww_mutex_lock(int tid)
 525 __acquires(torture_ww_mutex_0)
 526 __acquires(torture_ww_mutex_1)
 527 __acquires(torture_ww_mutex_2)
 528 {
 529         LIST_HEAD(list);
 530         struct reorder_lock {
 531                 struct list_head link;
 532                 struct ww_mutex *lock;
 533         } locks[3], *ll, *ln;
 534         struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
 535
 536         locks[0].lock = &torture_ww_mutex_0;
 537         list_add(&locks[0].link, &list);
 538
 539         locks[1].lock = &torture_ww_mutex_1;
 540         list_add(&locks[1].link, &list);
 541
 542         locks[2].lock = &torture_ww_mutex_2;
 543         list_add(&locks[2].link, &list);
 544
 545         ww_acquire_init(ctx, &torture_ww_class);
 546
 547         list_for_each_entry(ll, &list, link) {
 548                 int err;
 549
 550                 err = ww_mutex_lock(ll->lock, ctx);
 551                 if (!err)
 552                         continue;
 553
 554                 ln = ll;
 555                 list_for_each_entry_continue_reverse(ln, &list, link)
 556                         ww_mutex_unlock(ln->lock);
 557
 558                 if (err != -EDEADLK)
 559                         return err;
 560
 561                 ww_mutex_lock_slow(ll->lock, ctx);
 562                 list_move(&ll->link, &list);
 563         }
 564
 565         return 0;
 566 }
 567
 568 static void torture_ww_mutex_unlock(int tid)
 569 __releases(torture_ww_mutex_0)
 570 __releases(torture_ww_mutex_1)
 571 __releases(torture_ww_mutex_2)
 572 {
 573         struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
 574
 575         ww_mutex_unlock(&torture_ww_mutex_0);
 576         ww_mutex_unlock(&torture_ww_mutex_1);
 577         ww_mutex_unlock(&torture_ww_mutex_2);
 578         ww_acquire_fini(ctx);
 579 }
 580
 581 static struct lock_torture_ops ww_mutex_lock_ops = {
 582         .init           = torture_ww_mutex_init,
 583         .exit           = torture_ww_mutex_exit,
 584         .writelock      = torture_ww_mutex_lock,
 585         .write_delay    = torture_mutex_delay,
 586         .task_boost     = torture_rt_boost,
 587         .writeunlock    = torture_ww_mutex_unlock,
 588         .readlock       = NULL,
 589         .read_delay     = NULL,
 590         .readunlock     = NULL,
 591         .name           = "ww_mutex_lock"
 592 };
 593
 594 #ifdef CONFIG_RT_MUTEXES
 595 static DEFINE_RT_MUTEX(torture_rtmutex);
 596 static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS];
 597 static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS];
 598
 599 static void torture_rtmutex_init(void)
 600 {
 601         int i;
 602
 603         for (i = 0; i < MAX_NESTED_LOCKS; i++)
 604                 __rt_mutex_init(&torture_nested_rtmutexes[i], __func__,
 605                                 &nested_rtmutex_keys[i]);
 606 }
 607
 608 static int torture_rtmutex_nested_lock(int tid __maybe_unused,
 609                                        u32 lockset)
 610 {
 611         int i;
 612
 613         for (i = 0; i < nested_locks; i++)
 614                 if (lockset & (1 << i))
 615                         rt_mutex_lock(&torture_nested_rtmutexes[i]);
 616         return 0;
 617 }
 618
 619 static int torture_rtmutex_lock(int tid __maybe_unused)
 620 __acquires(torture_rtmutex)
 621 {
 622         rt_mutex_lock(&torture_rtmutex);
 623         return 0;
 624 }
 625
 626 static void torture_rtmutex_delay(struct torture_random_state *trsp)
 627 {
 628         const unsigned long shortdelay_us = 2;
 629         const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX;
 630
 631         /*
 632          * We want a short delay mostly to emulate likely code, and
 633          * we want a long delay occasionally to force massive contention.
 634          */
 635         if (!(torture_random(trsp) %
 636               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 637                 mdelay(longdelay_ms);
 638         if (!(torture_random(trsp) %
 639               (cxt.nrealwriters_stress * 200 * shortdelay_us)))
 640                 udelay(shortdelay_us);
 641         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 642                 torture_preempt_schedule();  /* Allow test to be preempted. */
 643 }
 644
 645 static void torture_rtmutex_unlock(int tid __maybe_unused)
 646 __releases(torture_rtmutex)
 647 {
 648         rt_mutex_unlock(&torture_rtmutex);
 649 }
 650
 651 static void torture_rt_boost_rtmutex(struct torture_random_state *trsp)
 652 {
 653         if (!rt_boost)
 654                 return;
 655
 656         __torture_rt_boost(trsp);
 657 }
 658
 659 static void torture_rtmutex_nested_unlock(int tid __maybe_unused,
 660                                           u32 lockset)
 661 {
 662         int i;
 663
 664         for (i = nested_locks - 1; i >= 0; i--)
 665                 if (lockset & (1 << i))
 666                         rt_mutex_unlock(&torture_nested_rtmutexes[i]);
 667 }
 668
 669 static struct lock_torture_ops rtmutex_lock_ops = {
 670         .init           = torture_rtmutex_init,
 671         .nested_lock    = torture_rtmutex_nested_lock,
 672         .writelock      = torture_rtmutex_lock,
 673         .write_delay    = torture_rtmutex_delay,
 674         .task_boost     = torture_rt_boost_rtmutex,
 675         .writeunlock    = torture_rtmutex_unlock,
 676         .nested_unlock  = torture_rtmutex_nested_unlock,
 677         .readlock       = NULL,
 678         .read_delay     = NULL,
 679         .readunlock     = NULL,
 680         .name           = "rtmutex_lock"
 681 };
 682 #endif
 683
 684 static DECLARE_RWSEM(torture_rwsem);
 685 static int torture_rwsem_down_write(int tid __maybe_unused)
 686 __acquires(torture_rwsem)
 687 {
 688         down_write(&torture_rwsem);
 689         return 0;
 690 }
 691
 692 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
 693 {
 694         const unsigned long longdelay_ms = long_hold ? long_hold : ULONG_MAX;
 695
 696         /* We want a long delay occasionally to force massive contention.  */
 697         if (!(torture_random(trsp) %
 698               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 699                 mdelay(longdelay_ms * 10);
 700         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 701                 torture_preempt_schedule();  /* Allow test to be preempted. */
 702 }
 703
 704 static void torture_rwsem_up_write(int tid __maybe_unused)
 705 __releases(torture_rwsem)
 706 {
 707         up_write(&torture_rwsem);
 708 }
 709
 710 static int torture_rwsem_down_read(int tid __maybe_unused)
 711 __acquires(torture_rwsem)
 712 {
 713         down_read(&torture_rwsem);
 714         return 0;
 715 }
 716
 717 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
 718 {
 719         const unsigned long longdelay_ms = 100;
 720
 721         /* We want a long delay occasionally to force massive contention.  */
 722         if (!(torture_random(trsp) %
 723               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 724                 mdelay(longdelay_ms * 2);
 725         else
 726                 mdelay(longdelay_ms / 2);
 727         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
 728                 torture_preempt_schedule();  /* Allow test to be preempted. */
 729 }
 730
 731 static void torture_rwsem_up_read(int tid __maybe_unused)
 732 __releases(torture_rwsem)
 733 {
 734         up_read(&torture_rwsem);
 735 }
 736
 737 static struct lock_torture_ops rwsem_lock_ops = {
 738         .writelock      = torture_rwsem_down_write,
 739         .write_delay    = torture_rwsem_write_delay,
 740         .task_boost     = torture_rt_boost,
 741         .writeunlock    = torture_rwsem_up_write,
 742         .readlock       = torture_rwsem_down_read,
 743         .read_delay     = torture_rwsem_read_delay,
 744         .readunlock     = torture_rwsem_up_read,
 745         .name           = "rwsem_lock"
 746 };
 747
 748 #include <linux/percpu-rwsem.h>
 749 static struct percpu_rw_semaphore pcpu_rwsem;
 750
 751 static void torture_percpu_rwsem_init(void)
 752 {
 753         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
 754 }
 755
 756 static void torture_percpu_rwsem_exit(void)
 757 {
 758         percpu_free_rwsem(&pcpu_rwsem);
 759 }
 760
 761 static int torture_percpu_rwsem_down_write(int tid __maybe_unused)
 762 __acquires(pcpu_rwsem)
 763 {
 764         percpu_down_write(&pcpu_rwsem);
 765         return 0;
 766 }
 767
 768 static void torture_percpu_rwsem_up_write(int tid __maybe_unused)
 769 __releases(pcpu_rwsem)
 770 {
 771         percpu_up_write(&pcpu_rwsem);
 772 }
 773
 774 static int torture_percpu_rwsem_down_read(int tid __maybe_unused)
 775 __acquires(pcpu_rwsem)
 776 {
 777         percpu_down_read(&pcpu_rwsem);
 778         return 0;
 779 }
 780
 781 static void torture_percpu_rwsem_up_read(int tid __maybe_unused)
 782 __releases(pcpu_rwsem)
 783 {
 784         percpu_up_read(&pcpu_rwsem);
 785 }
 786
 787 static struct lock_torture_ops percpu_rwsem_lock_ops = {
 788         .init           = torture_percpu_rwsem_init,
 789         .exit           = torture_percpu_rwsem_exit,
 790         .writelock      = torture_percpu_rwsem_down_write,
 791         .write_delay    = torture_rwsem_write_delay,
 792         .task_boost     = torture_rt_boost,
 793         .writeunlock    = torture_percpu_rwsem_up_write,
 794         .readlock       = torture_percpu_rwsem_down_read,
 795         .read_delay     = torture_rwsem_read_delay,
 796         .readunlock     = torture_percpu_rwsem_up_read,
 797         .name           = "percpu_rwsem_lock"
 798 };
 799
 800 /*
 801  * Lock torture writer kthread.  Repeatedly acquires and releases
 802  * the lock, checking for duplicate acquisitions.
 803  */
 804 static int lock_torture_writer(void *arg)
 805 {
 806         struct lock_stress_stats *lwsp = arg;
 807         int tid = lwsp - cxt.lwsa;
 808         DEFINE_TORTURE_RANDOM(rand);
 809         u32 lockset_mask;
 810         bool skip_main_lock;
 811
 812         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 813         if (!rt_task(current))
 814                 set_user_nice(current, MAX_NICE);
 815
 816         do {
 817                 if ((torture_random(&rand) & 0xfffff) == 0)
 818                         schedule_timeout_uninterruptible(1);
 819
 820                 lockset_mask = torture_random(&rand);
 821                 /*
 822                  * When using nested_locks, we want to occasionally
 823                  * skip the main lock so we can avoid always serializing
 824                  * the lock chains on that central lock. By skipping the
 825                  * main lock occasionally, we can create different
 826                  * contention patterns (allowing for multiple disjoint
 827                  * blocked trees)
 828                  */
 829                 skip_main_lock = (nested_locks &&
 830                                  !(torture_random(&rand) % 100));
 831
 832                 cxt.cur_ops->task_boost(&rand);
 833                 if (cxt.cur_ops->nested_lock)
 834                         cxt.cur_ops->nested_lock(tid, lockset_mask);
 835
 836                 if (!skip_main_lock) {
 837                         cxt.cur_ops->writelock(tid);
 838                         if (WARN_ON_ONCE(lock_is_write_held))
 839                                 lwsp->n_lock_fail++;
 840                         lock_is_write_held = true;
 841                         if (WARN_ON_ONCE(atomic_read(&lock_is_read_held)))
 842                                 lwsp->n_lock_fail++; /* rare, but... */
 843
 844                         lwsp->n_lock_acquired++;
 845                 }
 846                 if (!skip_main_lock) {
 847                         cxt.cur_ops->write_delay(&rand);
 848                         lock_is_write_held = false;
 849                         WRITE_ONCE(last_lock_release, jiffies);
 850                         cxt.cur_ops->writeunlock(tid);
 851                 }
 852                 if (cxt.cur_ops->nested_unlock)
 853                         cxt.cur_ops->nested_unlock(tid, lockset_mask);
 854
 855                 stutter_wait("lock_torture_writer");
 856         } while (!torture_must_stop());
 857
 858         cxt.cur_ops->task_boost(NULL); /* reset prio */
 859         torture_kthread_stopping("lock_torture_writer");
 860         return 0;
 861 }
 862
 863 /*
 864  * Lock torture reader kthread.  Repeatedly acquires and releases
 865  * the reader lock.
 866  */
 867 static int lock_torture_reader(void *arg)
 868 {
 869         struct lock_stress_stats *lrsp = arg;
 870         int tid = lrsp - cxt.lrsa;
 871         DEFINE_TORTURE_RANDOM(rand);
 872
 873         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 874         set_user_nice(current, MAX_NICE);
 875
 876         do {
 877                 if ((torture_random(&rand) & 0xfffff) == 0)
 878                         schedule_timeout_uninterruptible(1);
 879
 880                 cxt.cur_ops->readlock(tid);
 881                 atomic_inc(&lock_is_read_held);
 882                 if (WARN_ON_ONCE(lock_is_write_held))
 883                         lrsp->n_lock_fail++; /* rare, but... */
 884
 885                 lrsp->n_lock_acquired++;
 886                 cxt.cur_ops->read_delay(&rand);
 887                 atomic_dec(&lock_is_read_held);
 888                 cxt.cur_ops->readunlock(tid);
 889
 890                 stutter_wait("lock_torture_reader");
 891         } while (!torture_must_stop());
 892         torture_kthread_stopping("lock_torture_reader");
 893         return 0;
 894 }
 895
 896 /*
 897  * Create an lock-torture-statistics message in the specified buffer.
 898  */
 899 static void __torture_print_stats(char *page,
 900                                   struct lock_stress_stats *statp, bool write)
 901 {
 902         long cur;
 903         bool fail = false;
 904         int i, n_stress;
 905         long max = 0, min = statp ? data_race(statp[0].n_lock_acquired) : 0;
 906         long long sum = 0;
 907
 908         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
 909         for (i = 0; i < n_stress; i++) {
 910                 if (data_race(statp[i].n_lock_fail))
 911                         fail = true;
 912                 cur = data_race(statp[i].n_lock_acquired);
 913                 sum += cur;
 914                 if (max < cur)
 915                         max = cur;
 916                 if (min > cur)
 917                         min = cur;
 918         }
 919         page += sprintf(page,
 920                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 921                         write ? "Writes" : "Reads ",
 922                         sum, max, min,
 923                         !onoff_interval && max / 2 > min ? "???" : "",
 924                         fail, fail ? "!!!" : "");
 925         if (fail)
 926                 atomic_inc(&cxt.n_lock_torture_errors);
 927 }
 928
 929 /*
 930  * Print torture statistics.  Caller must ensure that there is only one
 931  * call to this function at a given time!!!  This is normally accomplished
 932  * by relying on the module system to only have one copy of the module
 933  * loaded, and then by giving the lock_torture_stats kthread full control
 934  * (or the init/cleanup functions when lock_torture_stats thread is not
 935  * running).
 936  */
 937 static void lock_torture_stats_print(void)
 938 {
 939         int size = cxt.nrealwriters_stress * 200 + 8192;
 940         char *buf;
 941
 942         if (cxt.cur_ops->readlock)
 943                 size += cxt.nrealreaders_stress * 200 + 8192;
 944
 945         buf = kmalloc(size, GFP_KERNEL);
 946         if (!buf) {
 947                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
 948                        size);
 949                 return;
 950         }
 951
 952         __torture_print_stats(buf, cxt.lwsa, true);
 953         pr_alert("%s", buf);
 954         kfree(buf);
 955
 956         if (cxt.cur_ops->readlock) {
 957                 buf = kmalloc(size, GFP_KERNEL);
 958                 if (!buf) {
 959                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
 960                                size);
 961                         return;
 962                 }
 963
 964                 __torture_print_stats(buf, cxt.lrsa, false);
 965                 pr_alert("%s", buf);
 966                 kfree(buf);
 967         }
 968 }
 969
 970 /*
 971  * Periodically prints torture statistics, if periodic statistics printing
 972  * was specified via the stat_interval module parameter.
 973  *
 974  * No need to worry about fullstop here, since this one doesn't reference
 975  * volatile state or register callbacks.
 976  */
 977 static int lock_torture_stats(void *arg)
 978 {
 979         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
 980         do {
 981                 schedule_timeout_interruptible(stat_interval * HZ);
 982                 lock_torture_stats_print();
 983                 torture_shutdown_absorb("lock_torture_stats");
 984         } while (!torture_must_stop());
 985         torture_kthread_stopping("lock_torture_stats");
 986         return 0;
 987 }
 988
 989 static inline void
 990 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
 991                                 const char *tag)
 992 {
 993         pr_alert("%s" TORTURE_FLAG
 994                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d nested_locks=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
 995                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
 996                  cxt.nrealwriters_stress, cxt.nrealreaders_stress,
 997                  nested_locks, stat_interval, verbose, shuffle_interval,
 998                  stutter, shutdown_secs, onoff_interval, onoff_holdoff);
 999 }
1000
1001 static void lock_torture_cleanup(void)
1002 {
1003         int i;
1004
1005         if (torture_cleanup_begin())
1006                 return;
1007
1008         /*
1009          * Indicates early cleanup, meaning that the test has not run,
1010          * such as when passing bogus args when loading the module.
1011          * However cxt->cur_ops.init() may have been invoked, so beside
1012          * perform the underlying torture-specific cleanups, cur_ops.exit()
1013          * will be invoked if needed.
1014          */
1015         if (!cxt.lwsa && !cxt.lrsa)
1016                 goto end;
1017
1018         if (writer_tasks) {
1019                 for (i = 0; i < cxt.nrealwriters_stress; i++)
1020                         torture_stop_kthread(lock_torture_writer, writer_tasks[i]);
1021                 kfree(writer_tasks);
1022                 writer_tasks = NULL;
1023         }
1024
1025         if (reader_tasks) {
1026                 for (i = 0; i < cxt.nrealreaders_stress; i++)
1027                         torture_stop_kthread(lock_torture_reader,
1028                                              reader_tasks[i]);
1029                 kfree(reader_tasks);
1030                 reader_tasks = NULL;
1031         }
1032
1033         torture_stop_kthread(lock_torture_stats, stats_task);
1034         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
1035
1036         if (atomic_read(&cxt.n_lock_torture_errors))
1037                 lock_torture_print_module_parms(cxt.cur_ops,
1038                                                 "End of test: FAILURE");
1039         else if (torture_onoff_failures())
1040                 lock_torture_print_module_parms(cxt.cur_ops,
1041                                                 "End of test: LOCK_HOTPLUG");
1042         else
1043                 lock_torture_print_module_parms(cxt.cur_ops,
1044                                                 "End of test: SUCCESS");
1045
1046         kfree(cxt.lwsa);
1047         cxt.lwsa = NULL;
1048         kfree(cxt.lrsa);
1049         cxt.lrsa = NULL;
1050
1051 end:
1052         if (cxt.init_called) {
1053                 if (cxt.cur_ops->exit)
1054                         cxt.cur_ops->exit();
1055                 cxt.init_called = false;
1056         }
1057         torture_cleanup_end();
1058 }
1059
1060 static int __init lock_torture_init(void)
1061 {
1062         int i, j;
1063         int firsterr = 0;
1064         static struct lock_torture_ops *torture_ops[] = {
1065                 &lock_busted_ops,
1066                 &spin_lock_ops, &spin_lock_irq_ops,
1067                 &raw_spin_lock_ops, &raw_spin_lock_irq_ops,
1068                 &rw_lock_ops, &rw_lock_irq_ops,
1069                 &mutex_lock_ops,
1070                 &ww_mutex_lock_ops,
1071 #ifdef CONFIG_RT_MUTEXES
1072                 &rtmutex_lock_ops,
1073 #endif
1074                 &rwsem_lock_ops,
1075                 &percpu_rwsem_lock_ops,
1076         };
1077
1078         if (!torture_init_begin(torture_type, verbose))
1079                 return -EBUSY;
1080
1081         /* Process args and tell the world that the torturer is on the job. */
1082         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1083                 cxt.cur_ops = torture_ops[i];
1084                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
1085                         break;
1086         }
1087         if (i == ARRAY_SIZE(torture_ops)) {
1088                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
1089                          torture_type);
1090                 pr_alert("lock-torture types:");
1091                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1092                         pr_alert(" %s", torture_ops[i]->name);
1093                 pr_alert("\n");
1094                 firsterr = -EINVAL;
1095                 goto unwind;
1096         }
1097
1098         if (nwriters_stress == 0 &&
1099             (!cxt.cur_ops->readlock || nreaders_stress == 0)) {
1100                 pr_alert("lock-torture: must run at least one locking thread\n");
1101                 firsterr = -EINVAL;
1102                 goto unwind;
1103         }
1104
1105         if (nwriters_stress >= 0)
1106                 cxt.nrealwriters_stress = nwriters_stress;
1107         else
1108                 cxt.nrealwriters_stress = 2 * num_online_cpus();
1109
1110         if (cxt.cur_ops->init) {
1111                 cxt.cur_ops->init();
1112                 cxt.init_called = true;
1113         }
1114
1115 #ifdef CONFIG_DEBUG_MUTEXES
1116         if (str_has_prefix(torture_type, "mutex"))
1117                 cxt.debug_lock = true;
1118 #endif
1119 #ifdef CONFIG_DEBUG_RT_MUTEXES
1120         if (str_has_prefix(torture_type, "rtmutex"))
1121                 cxt.debug_lock = true;
1122 #endif
1123 #ifdef CONFIG_DEBUG_SPINLOCK
1124         if ((str_has_prefix(torture_type, "spin")) ||
1125             (str_has_prefix(torture_type, "rw_lock")))
1126                 cxt.debug_lock = true;
1127 #endif
1128
1129         /* Initialize the statistics so that each run gets its own numbers. */
1130         if (nwriters_stress) {
1131                 lock_is_write_held = false;
1132                 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
1133                                          sizeof(*cxt.lwsa),
1134                                          GFP_KERNEL);
1135                 if (cxt.lwsa == NULL) {
1136                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
1137                         firsterr = -ENOMEM;
1138                         goto unwind;
1139                 }
1140
1141                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
1142                         cxt.lwsa[i].n_lock_fail = 0;
1143                         cxt.lwsa[i].n_lock_acquired = 0;
1144                 }
1145         }
1146
1147         if (cxt.cur_ops->readlock) {
1148                 if (nreaders_stress >= 0)
1149                         cxt.nrealreaders_stress = nreaders_stress;
1150                 else {
1151                         /*
1152                          * By default distribute evenly the number of
1153                          * readers and writers. We still run the same number
1154                          * of threads as the writer-only locks default.
1155                          */
1156                         if (nwriters_stress < 0) /* user doesn't care */
1157                                 cxt.nrealwriters_stress = num_online_cpus();
1158                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
1159                 }
1160
1161                 if (nreaders_stress) {
1162                         cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
1163                                                  sizeof(*cxt.lrsa),
1164                                                  GFP_KERNEL);
1165                         if (cxt.lrsa == NULL) {
1166                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
1167                                 firsterr = -ENOMEM;
1168                                 kfree(cxt.lwsa);
1169                                 cxt.lwsa = NULL;
1170                                 goto unwind;
1171                         }
1172
1173                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
1174                                 cxt.lrsa[i].n_lock_fail = 0;
1175                                 cxt.lrsa[i].n_lock_acquired = 0;
1176                         }
1177                 }
1178         }
1179
1180         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
1181
1182         /* Prepare torture context. */
1183         if (onoff_interval > 0) {
1184                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
1185                                               onoff_interval * HZ, NULL);
1186                 if (torture_init_error(firsterr))
1187                         goto unwind;
1188         }
1189         if (shuffle_interval > 0) {
1190                 firsterr = torture_shuffle_init(shuffle_interval);
1191                 if (torture_init_error(firsterr))
1192                         goto unwind;
1193         }
1194         if (shutdown_secs > 0) {
1195                 firsterr = torture_shutdown_init(shutdown_secs,
1196                                                  lock_torture_cleanup);
1197                 if (torture_init_error(firsterr))
1198                         goto unwind;
1199         }
1200         if (stutter > 0) {
1201                 firsterr = torture_stutter_init(stutter, stutter);
1202                 if (torture_init_error(firsterr))
1203                         goto unwind;
1204         }
1205
1206         if (nwriters_stress) {
1207                 writer_tasks = kcalloc(cxt.nrealwriters_stress,
1208                                        sizeof(writer_tasks[0]),
1209                                        GFP_KERNEL);
1210                 if (writer_tasks == NULL) {
1211                         TOROUT_ERRSTRING("writer_tasks: Out of memory");
1212                         firsterr = -ENOMEM;
1213                         goto unwind;
1214                 }
1215         }
1216
1217         /* cap nested_locks to MAX_NESTED_LOCKS */
1218         if (nested_locks > MAX_NESTED_LOCKS)
1219                 nested_locks = MAX_NESTED_LOCKS;
1220
1221         if (cxt.cur_ops->readlock) {
1222                 reader_tasks = kcalloc(cxt.nrealreaders_stress,
1223                                        sizeof(reader_tasks[0]),
1224                                        GFP_KERNEL);
1225                 if (reader_tasks == NULL) {
1226                         TOROUT_ERRSTRING("reader_tasks: Out of memory");
1227                         kfree(writer_tasks);
1228                         writer_tasks = NULL;
1229                         firsterr = -ENOMEM;
1230                         goto unwind;
1231                 }
1232         }
1233
1234         /*
1235          * Create the kthreads and start torturing (oh, those poor little locks).
1236          *
1237          * TODO: Note that we interleave writers with readers, giving writers a
1238          * slight advantage, by creating its kthread first. This can be modified
1239          * for very specific needs, or even let the user choose the policy, if
1240          * ever wanted.
1241          */
1242         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1243                     j < cxt.nrealreaders_stress; i++, j++) {
1244                 if (i >= cxt.nrealwriters_stress)
1245                         goto create_reader;
1246
1247                 /* Create writer. */
1248                 firsterr = torture_create_kthread_cb(lock_torture_writer, &cxt.lwsa[i],
1249                                                      writer_tasks[i],
1250                                                      writer_fifo ? sched_set_fifo : NULL);
1251                 if (torture_init_error(firsterr))
1252                         goto unwind;
1253
1254         create_reader:
1255                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1256                         continue;
1257                 /* Create reader. */
1258                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1259                                                   reader_tasks[j]);
1260                 if (torture_init_error(firsterr))
1261                         goto unwind;
1262         }
1263         if (stat_interval > 0) {
1264                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1265                                                   stats_task);
1266                 if (torture_init_error(firsterr))
1267                         goto unwind;
1268         }
1269         torture_init_end();
1270         return 0;
1271
1272 unwind:
1273         torture_init_end();
1274         lock_torture_cleanup();
1275         if (shutdown_secs) {
1276                 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST));
1277                 kernel_power_off();
1278         }
1279         return firsterr;
1280 }
1281
1282 module_init(lock_torture_init);
1283 module_exit(lock_torture_cleanup);