kernel/rcutorture.c

   1 /*
   2  * Read-Copy Update module-based torture test facility
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2005, 2006
  19  *
  20  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21  *          Josh Triplett <josh@freedesktop.org>
  22  *
  23  * See also:  Documentation/RCU/torture.txt
  24  */
  25 #include <linux/types.h>
  26 #include <linux/kernel.h>
  27 #include <linux/init.h>
  28 #include <linux/module.h>
  29 #include <linux/kthread.h>
  30 #include <linux/err.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/smp.h>
  33 #include <linux/rcupdate.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/sched.h>
  36 #include <asm/atomic.h>
  37 #include <linux/bitops.h>
  38 #include <linux/completion.h>
  39 #include <linux/moduleparam.h>
  40 #include <linux/percpu.h>
  41 #include <linux/notifier.h>
  42 #include <linux/reboot.h>
  43 #include <linux/freezer.h>
  44 #include <linux/cpu.h>
  45 #include <linux/delay.h>
  46 #include <linux/stat.h>
  47 #include <linux/srcu.h>
  48 #include <linux/slab.h>
  49 #include <asm/byteorder.h>
  50
  51 MODULE_LICENSE("GPL");
  52 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
  53               "Josh Triplett <josh@freedesktop.org>");
  54
  55 static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
  56 static int nfakewriters = 4;    /* # fake writer threads */
  57 static int stat_interval;       /* Interval between stats, in seconds. */
  58                                 /*  Defaults to "only at end of test". */
  59 static int verbose;             /* Print more debug info. */
  60 static int test_no_idle_hz;     /* Test RCU's support for tickless idle CPUs. */
  61 static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
  62 static int stutter = 5;         /* Start/stop testing interval (in sec) */
  63 static int irqreader = 1;       /* RCU readers from irq (timers). */
  64 static char *torture_type = "rcu"; /* What RCU implementation to torture. */
  65
  66 module_param(nreaders, int, 0444);
  67 MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  68 module_param(nfakewriters, int, 0444);
  69 MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
  70 module_param(stat_interval, int, 0444);
  71 MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  72 module_param(verbose, bool, 0444);
  73 MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  74 module_param(test_no_idle_hz, bool, 0444);
  75 MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  76 module_param(shuffle_interval, int, 0444);
  77 MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  78 module_param(stutter, int, 0444);
  79 MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
  80 module_param(irqreader, int, 0444);
  81 MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
  82 module_param(torture_type, charp, 0444);
  83 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
  84
  85 #define TORTURE_FLAG "-torture:"
  86 #define PRINTK_STRING(s) \
  87         do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  88 #define VERBOSE_PRINTK_STRING(s) \
  89         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  90 #define VERBOSE_PRINTK_ERRSTRING(s) \
  91         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
  92
  93 static char printk_buf[4096];
  94
  95 static int nrealreaders;
  96 static struct task_struct *writer_task;
  97 static struct task_struct **fakewriter_tasks;
  98 static struct task_struct **reader_tasks;
  99 static struct task_struct *stats_task;
 100 static struct task_struct *shuffler_task;
 101 static struct task_struct *stutter_task;
 102
 103 #define RCU_TORTURE_PIPE_LEN 10
 104
 105 struct rcu_torture {
 106         struct rcu_head rtort_rcu;
 107         int rtort_pipe_count;
 108         struct list_head rtort_free;
 109         int rtort_mbtest;
 110 };
 111
 112 static LIST_HEAD(rcu_torture_freelist);
 113 static struct rcu_torture *rcu_torture_current = NULL;
 114 static long rcu_torture_current_version = 0;
 115 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 116 static DEFINE_SPINLOCK(rcu_torture_lock);
 117 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
 118         { 0 };
 119 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
 120         { 0 };
 121 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 122 static atomic_t n_rcu_torture_alloc;
 123 static atomic_t n_rcu_torture_alloc_fail;
 124 static atomic_t n_rcu_torture_free;
 125 static atomic_t n_rcu_torture_mberror;
 126 static atomic_t n_rcu_torture_error;
 127 static long n_rcu_torture_timers = 0;
 128 static struct list_head rcu_torture_removed;
 129
 130 static int stutter_pause_test = 0;
 131
 132 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
 133 #define RCUTORTURE_RUNNABLE_INIT 1
 134 #else
 135 #define RCUTORTURE_RUNNABLE_INIT 0
 136 #endif
 137 int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
 138
 139 #define FULLSTOP_SIGNALED 1     /* Bail due to signal. */
 140 #define FULLSTOP_CLEANUP  2     /* Orderly shutdown. */
 141 static int fullstop;            /* stop generating callbacks at test end. */
 142 DEFINE_MUTEX(fullstop_mutex);   /* protect fullstop transitions and */
 143                                 /*  spawning of kthreads. */
 144
 145 /*
 146  * Detect and respond to a signal-based shutdown.
 147  */
 148 static int
 149 rcutorture_shutdown_notify(struct notifier_block *unused1,
 150                            unsigned long unused2, void *unused3)
 151 {
 152         if (fullstop)
 153                 return NOTIFY_DONE;
 154         if (signal_pending(current)) {
 155                 mutex_lock(&fullstop_mutex);
 156                 if (!ACCESS_ONCE(fullstop))
 157                         fullstop = FULLSTOP_SIGNALED;
 158                 mutex_unlock(&fullstop_mutex);
 159         }
 160         return NOTIFY_DONE;
 161 }
 162
 163 /*
 164  * Allocate an element from the rcu_tortures pool.
 165  */
 166 static struct rcu_torture *
 167 rcu_torture_alloc(void)
 168 {
 169         struct list_head *p;
 170
 171         spin_lock_bh(&rcu_torture_lock);
 172         if (list_empty(&rcu_torture_freelist)) {
 173                 atomic_inc(&n_rcu_torture_alloc_fail);
 174                 spin_unlock_bh(&rcu_torture_lock);
 175                 return NULL;
 176         }
 177         atomic_inc(&n_rcu_torture_alloc);
 178         p = rcu_torture_freelist.next;
 179         list_del_init(p);
 180         spin_unlock_bh(&rcu_torture_lock);
 181         return container_of(p, struct rcu_torture, rtort_free);
 182 }
 183
 184 /*
 185  * Free an element to the rcu_tortures pool.
 186  */
 187 static void
 188 rcu_torture_free(struct rcu_torture *p)
 189 {
 190         atomic_inc(&n_rcu_torture_free);
 191         spin_lock_bh(&rcu_torture_lock);
 192         list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 193         spin_unlock_bh(&rcu_torture_lock);
 194 }
 195
 196 struct rcu_random_state {
 197         unsigned long rrs_state;
 198         long rrs_count;
 199 };
 200
 201 #define RCU_RANDOM_MULT 39916801  /* prime */
 202 #define RCU_RANDOM_ADD  479001701 /* prime */
 203 #define RCU_RANDOM_REFRESH 10000
 204
 205 #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
 206
 207 /*
 208  * Crude but fast random-number generator.  Uses a linear congruential
 209  * generator, with occasional help from cpu_clock().
 210  */
 211 static unsigned long
 212 rcu_random(struct rcu_random_state *rrsp)
 213 {
 214         if (--rrsp->rrs_count < 0) {
 215                 rrsp->rrs_state +=
 216                         (unsigned long)cpu_clock(raw_smp_processor_id());
 217                 rrsp->rrs_count = RCU_RANDOM_REFRESH;
 218         }
 219         rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
 220         return swahw32(rrsp->rrs_state);
 221 }
 222
 223 static void
 224 rcu_stutter_wait(void)
 225 {
 226         while ((stutter_pause_test || !rcutorture_runnable) && !fullstop) {
 227                 if (rcutorture_runnable)
 228                         schedule_timeout_interruptible(1);
 229                 else
 230                         schedule_timeout_interruptible(round_jiffies_relative(HZ));
 231         }
 232 }
 233
 234 /*
 235  * Operations vector for selecting different types of tests.
 236  */
 237
 238 struct rcu_torture_ops {
 239         void (*init)(void);
 240         void (*cleanup)(void);
 241         int (*readlock)(void);
 242         void (*readdelay)(struct rcu_random_state *rrsp);
 243         void (*readunlock)(int idx);
 244         int (*completed)(void);
 245         void (*deferredfree)(struct rcu_torture *p);
 246         void (*sync)(void);
 247         void (*cb_barrier)(void);
 248         int (*stats)(char *page);
 249         int irqcapable;
 250         char *name;
 251 };
 252 static struct rcu_torture_ops *cur_ops = NULL;
 253
 254 /*
 255  * Definitions for rcu torture testing.
 256  */
 257
 258 static int rcu_torture_read_lock(void) __acquires(RCU)
 259 {
 260         rcu_read_lock();
 261         return 0;
 262 }
 263
 264 static void rcu_read_delay(struct rcu_random_state *rrsp)
 265 {
 266         long delay;
 267         const long longdelay = 200;
 268
 269         /* We want there to be long-running readers, but not all the time. */
 270
 271         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay);
 272         if (!delay)
 273                 udelay(longdelay);
 274 }
 275
 276 static void rcu_torture_read_unlock(int idx) __releases(RCU)
 277 {
 278         rcu_read_unlock();
 279 }
 280
 281 static int rcu_torture_completed(void)
 282 {
 283         return rcu_batches_completed();
 284 }
 285
 286 static void
 287 rcu_torture_cb(struct rcu_head *p)
 288 {
 289         int i;
 290         struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 291
 292         if (fullstop) {
 293                 /* Test is ending, just drop callbacks on the floor. */
 294                 /* The next initialization will pick up the pieces. */
 295                 return;
 296         }
 297         i = rp->rtort_pipe_count;
 298         if (i > RCU_TORTURE_PIPE_LEN)
 299                 i = RCU_TORTURE_PIPE_LEN;
 300         atomic_inc(&rcu_torture_wcount[i]);
 301         if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 302                 rp->rtort_mbtest = 0;
 303                 rcu_torture_free(rp);
 304         } else
 305                 cur_ops->deferredfree(rp);
 306 }
 307
 308 static void rcu_torture_deferred_free(struct rcu_torture *p)
 309 {
 310         call_rcu(&p->rtort_rcu, rcu_torture_cb);
 311 }
 312
 313 static struct rcu_torture_ops rcu_ops = {
 314         .init = NULL,
 315         .cleanup = NULL,
 316         .readlock = rcu_torture_read_lock,
 317         .readdelay = rcu_read_delay,
 318         .readunlock = rcu_torture_read_unlock,
 319         .completed = rcu_torture_completed,
 320         .deferredfree = rcu_torture_deferred_free,
 321         .sync = synchronize_rcu,
 322         .cb_barrier = rcu_barrier,
 323         .stats = NULL,
 324         .irqcapable = 1,
 325         .name = "rcu"
 326 };
 327
 328 static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
 329 {
 330         int i;
 331         struct rcu_torture *rp;
 332         struct rcu_torture *rp1;
 333
 334         cur_ops->sync();
 335         list_add(&p->rtort_free, &rcu_torture_removed);
 336         list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
 337                 i = rp->rtort_pipe_count;
 338                 if (i > RCU_TORTURE_PIPE_LEN)
 339                         i = RCU_TORTURE_PIPE_LEN;
 340                 atomic_inc(&rcu_torture_wcount[i]);
 341                 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 342                         rp->rtort_mbtest = 0;
 343                         list_del(&rp->rtort_free);
 344                         rcu_torture_free(rp);
 345                 }
 346         }
 347 }
 348
 349 static void rcu_sync_torture_init(void)
 350 {
 351         INIT_LIST_HEAD(&rcu_torture_removed);
 352 }
 353
 354 static struct rcu_torture_ops rcu_sync_ops = {
 355         .init = rcu_sync_torture_init,
 356         .cleanup = NULL,
 357         .readlock = rcu_torture_read_lock,
 358         .readdelay = rcu_read_delay,
 359         .readunlock = rcu_torture_read_unlock,
 360         .completed = rcu_torture_completed,
 361         .deferredfree = rcu_sync_torture_deferred_free,
 362         .sync = synchronize_rcu,
 363         .cb_barrier = NULL,
 364         .stats = NULL,
 365         .irqcapable = 1,
 366         .name = "rcu_sync"
 367 };
 368
 369 /*
 370  * Definitions for rcu_bh torture testing.
 371  */
 372
 373 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
 374 {
 375         rcu_read_lock_bh();
 376         return 0;
 377 }
 378
 379 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
 380 {
 381         rcu_read_unlock_bh();
 382 }
 383
 384 static int rcu_bh_torture_completed(void)
 385 {
 386         return rcu_batches_completed_bh();
 387 }
 388
 389 static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 390 {
 391         call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
 392 }
 393
 394 struct rcu_bh_torture_synchronize {
 395         struct rcu_head head;
 396         struct completion completion;
 397 };
 398
 399 static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head)
 400 {
 401         struct rcu_bh_torture_synchronize *rcu;
 402
 403         rcu = container_of(head, struct rcu_bh_torture_synchronize, head);
 404         complete(&rcu->completion);
 405 }
 406
 407 static void rcu_bh_torture_synchronize(void)
 408 {
 409         struct rcu_bh_torture_synchronize rcu;
 410
 411         init_completion(&rcu.completion);
 412         call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);
 413         wait_for_completion(&rcu.completion);
 414 }
 415
 416 static struct rcu_torture_ops rcu_bh_ops = {
 417         .init = NULL,
 418         .cleanup = NULL,
 419         .readlock = rcu_bh_torture_read_lock,
 420         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
 421         .readunlock = rcu_bh_torture_read_unlock,
 422         .completed = rcu_bh_torture_completed,
 423         .deferredfree = rcu_bh_torture_deferred_free,
 424         .sync = rcu_bh_torture_synchronize,
 425         .cb_barrier = rcu_barrier_bh,
 426         .stats = NULL,
 427         .irqcapable = 1,
 428         .name = "rcu_bh"
 429 };
 430
 431 static struct rcu_torture_ops rcu_bh_sync_ops = {
 432         .init = rcu_sync_torture_init,
 433         .cleanup = NULL,
 434         .readlock = rcu_bh_torture_read_lock,
 435         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
 436         .readunlock = rcu_bh_torture_read_unlock,
 437         .completed = rcu_bh_torture_completed,
 438         .deferredfree = rcu_sync_torture_deferred_free,
 439         .sync = rcu_bh_torture_synchronize,
 440         .cb_barrier = NULL,
 441         .stats = NULL,
 442         .irqcapable = 1,
 443         .name = "rcu_bh_sync"
 444 };
 445
 446 /*
 447  * Definitions for srcu torture testing.
 448  */
 449
 450 static struct srcu_struct srcu_ctl;
 451
 452 static void srcu_torture_init(void)
 453 {
 454         init_srcu_struct(&srcu_ctl);
 455         rcu_sync_torture_init();
 456 }
 457
 458 static void srcu_torture_cleanup(void)
 459 {
 460         synchronize_srcu(&srcu_ctl);
 461         cleanup_srcu_struct(&srcu_ctl);
 462 }
 463
 464 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
 465 {
 466         return srcu_read_lock(&srcu_ctl);
 467 }
 468
 469 static void srcu_read_delay(struct rcu_random_state *rrsp)
 470 {
 471         long delay;
 472         const long uspertick = 1000000 / HZ;
 473         const long longdelay = 10;
 474
 475         /* We want there to be long-running readers, but not all the time. */
 476
 477         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
 478         if (!delay)
 479                 schedule_timeout_interruptible(longdelay);
 480 }
 481
 482 static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
 483 {
 484         srcu_read_unlock(&srcu_ctl, idx);
 485 }
 486
 487 static int srcu_torture_completed(void)
 488 {
 489         return srcu_batches_completed(&srcu_ctl);
 490 }
 491
 492 static void srcu_torture_synchronize(void)
 493 {
 494         synchronize_srcu(&srcu_ctl);
 495 }
 496
 497 static int srcu_torture_stats(char *page)
 498 {
 499         int cnt = 0;
 500         int cpu;
 501         int idx = srcu_ctl.completed & 0x1;
 502
 503         cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
 504                        torture_type, TORTURE_FLAG, idx);
 505         for_each_possible_cpu(cpu) {
 506                 cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
 507                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
 508                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
 509         }
 510         cnt += sprintf(&page[cnt], "\n");
 511         return cnt;
 512 }
 513
 514 static struct rcu_torture_ops srcu_ops = {
 515         .init = srcu_torture_init,
 516         .cleanup = srcu_torture_cleanup,
 517         .readlock = srcu_torture_read_lock,
 518         .readdelay = srcu_read_delay,
 519         .readunlock = srcu_torture_read_unlock,
 520         .completed = srcu_torture_completed,
 521         .deferredfree = rcu_sync_torture_deferred_free,
 522         .sync = srcu_torture_synchronize,
 523         .cb_barrier = NULL,
 524         .stats = srcu_torture_stats,
 525         .name = "srcu"
 526 };
 527
 528 /*
 529  * Definitions for sched torture testing.
 530  */
 531
 532 static int sched_torture_read_lock(void)
 533 {
 534         preempt_disable();
 535         return 0;
 536 }
 537
 538 static void sched_torture_read_unlock(int idx)
 539 {
 540         preempt_enable();
 541 }
 542
 543 static int sched_torture_completed(void)
 544 {
 545         return 0;
 546 }
 547
 548 static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
 549 {
 550         call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
 551 }
 552
 553 static void sched_torture_synchronize(void)
 554 {
 555         synchronize_sched();
 556 }
 557
 558 static struct rcu_torture_ops sched_ops = {
 559         .init = rcu_sync_torture_init,
 560         .cleanup = NULL,
 561         .readlock = sched_torture_read_lock,
 562         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
 563         .readunlock = sched_torture_read_unlock,
 564         .completed = sched_torture_completed,
 565         .deferredfree = rcu_sched_torture_deferred_free,
 566         .sync = sched_torture_synchronize,
 567         .cb_barrier = rcu_barrier_sched,
 568         .stats = NULL,
 569         .irqcapable = 1,
 570         .name = "sched"
 571 };
 572
 573 static struct rcu_torture_ops sched_ops_sync = {
 574         .init = rcu_sync_torture_init,
 575         .cleanup = NULL,
 576         .readlock = sched_torture_read_lock,
 577         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
 578         .readunlock = sched_torture_read_unlock,
 579         .completed = sched_torture_completed,
 580         .deferredfree = rcu_sync_torture_deferred_free,
 581         .sync = sched_torture_synchronize,
 582         .cb_barrier = NULL,
 583         .stats = NULL,
 584         .name = "sched_sync"
 585 };
 586
 587 /*
 588  * RCU torture writer kthread.  Repeatedly substitutes a new structure
 589  * for that pointed to by rcu_torture_current, freeing the old structure
 590  * after a series of grace periods (the "pipeline").
 591  */
 592 static int
 593 rcu_torture_writer(void *arg)
 594 {
 595         int i;
 596         long oldbatch = rcu_batches_completed();
 597         struct rcu_torture *rp;
 598         struct rcu_torture *old_rp;
 599         static DEFINE_RCU_RANDOM(rand);
 600
 601         VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
 602         set_user_nice(current, 19);
 603
 604         do {
 605                 schedule_timeout_uninterruptible(1);
 606                 if ((rp = rcu_torture_alloc()) == NULL)
 607                         continue;
 608                 rp->rtort_pipe_count = 0;
 609                 udelay(rcu_random(&rand) & 0x3ff);
 610                 old_rp = rcu_torture_current;
 611                 rp->rtort_mbtest = 1;
 612                 rcu_assign_pointer(rcu_torture_current, rp);
 613                 smp_wmb();
 614                 if (old_rp) {
 615                         i = old_rp->rtort_pipe_count;
 616                         if (i > RCU_TORTURE_PIPE_LEN)
 617                                 i = RCU_TORTURE_PIPE_LEN;
 618                         atomic_inc(&rcu_torture_wcount[i]);
 619                         old_rp->rtort_pipe_count++;
 620                         cur_ops->deferredfree(old_rp);
 621                 }
 622                 rcu_torture_current_version++;
 623                 oldbatch = cur_ops->completed();
 624                 rcu_stutter_wait();
 625         } while (!kthread_should_stop() && !fullstop);
 626         VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
 627         while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
 628                 schedule_timeout_uninterruptible(1);
 629         return 0;
 630 }
 631
 632 /*
 633  * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
 634  * delay between calls.
 635  */
 636 static int
 637 rcu_torture_fakewriter(void *arg)
 638 {
 639         DEFINE_RCU_RANDOM(rand);
 640
 641         VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
 642         set_user_nice(current, 19);
 643
 644         do {
 645                 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
 646                 udelay(rcu_random(&rand) & 0x3ff);
 647                 cur_ops->sync();
 648                 rcu_stutter_wait();
 649         } while (!kthread_should_stop() && !fullstop);
 650
 651         VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
 652         while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
 653                 schedule_timeout_uninterruptible(1);
 654         return 0;
 655 }
 656
 657 /*
 658  * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
 659  * incrementing the corresponding element of the pipeline array.  The
 660  * counter in the element should never be greater than 1, otherwise, the
 661  * RCU implementation is broken.
 662  */
 663 static void rcu_torture_timer(unsigned long unused)
 664 {
 665         int idx;
 666         int completed;
 667         static DEFINE_RCU_RANDOM(rand);
 668         static DEFINE_SPINLOCK(rand_lock);
 669         struct rcu_torture *p;
 670         int pipe_count;
 671
 672         idx = cur_ops->readlock();
 673         completed = cur_ops->completed();
 674         p = rcu_dereference(rcu_torture_current);
 675         if (p == NULL) {
 676                 /* Leave because rcu_torture_writer is not yet underway */
 677                 cur_ops->readunlock(idx);
 678                 return;
 679         }
 680         if (p->rtort_mbtest == 0)
 681                 atomic_inc(&n_rcu_torture_mberror);
 682         spin_lock(&rand_lock);
 683         cur_ops->readdelay(&rand);
 684         n_rcu_torture_timers++;
 685         spin_unlock(&rand_lock);
 686         preempt_disable();
 687         pipe_count = p->rtort_pipe_count;
 688         if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 689                 /* Should not happen, but... */
 690                 pipe_count = RCU_TORTURE_PIPE_LEN;
 691         }
 692         ++__get_cpu_var(rcu_torture_count)[pipe_count];
 693         completed = cur_ops->completed() - completed;
 694         if (completed > RCU_TORTURE_PIPE_LEN) {
 695                 /* Should not happen, but... */
 696                 completed = RCU_TORTURE_PIPE_LEN;
 697         }
 698         ++__get_cpu_var(rcu_torture_batch)[completed];
 699         preempt_enable();
 700         cur_ops->readunlock(idx);
 701 }
 702
 703 /*
 704  * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 705  * incrementing the corresponding element of the pipeline array.  The
 706  * counter in the element should never be greater than 1, otherwise, the
 707  * RCU implementation is broken.
 708  */
 709 static int
 710 rcu_torture_reader(void *arg)
 711 {
 712         int completed;
 713         int idx;
 714         DEFINE_RCU_RANDOM(rand);
 715         struct rcu_torture *p;
 716         int pipe_count;
 717         struct timer_list t;
 718
 719         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
 720         set_user_nice(current, 19);
 721         if (irqreader && cur_ops->irqcapable)
 722                 setup_timer_on_stack(&t, rcu_torture_timer, 0);
 723
 724         do {
 725                 if (irqreader && cur_ops->irqcapable) {
 726                         if (!timer_pending(&t))
 727                                 mod_timer(&t, 1);
 728                 }
 729                 idx = cur_ops->readlock();
 730                 completed = cur_ops->completed();
 731                 p = rcu_dereference(rcu_torture_current);
 732                 if (p == NULL) {
 733                         /* Wait for rcu_torture_writer to get underway */
 734                         cur_ops->readunlock(idx);
 735                         schedule_timeout_interruptible(HZ);
 736                         continue;
 737                 }
 738                 if (p->rtort_mbtest == 0)
 739                         atomic_inc(&n_rcu_torture_mberror);
 740                 cur_ops->readdelay(&rand);
 741                 preempt_disable();
 742                 pipe_count = p->rtort_pipe_count;
 743                 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 744                         /* Should not happen, but... */
 745                         pipe_count = RCU_TORTURE_PIPE_LEN;
 746                 }
 747                 ++__get_cpu_var(rcu_torture_count)[pipe_count];
 748                 completed = cur_ops->completed() - completed;
 749                 if (completed > RCU_TORTURE_PIPE_LEN) {
 750                         /* Should not happen, but... */
 751                         completed = RCU_TORTURE_PIPE_LEN;
 752                 }
 753                 ++__get_cpu_var(rcu_torture_batch)[completed];
 754                 preempt_enable();
 755                 cur_ops->readunlock(idx);
 756                 schedule();
 757                 rcu_stutter_wait();
 758         } while (!kthread_should_stop() && !fullstop);
 759         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
 760         if (irqreader && cur_ops->irqcapable)
 761                 del_timer_sync(&t);
 762         while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
 763                 schedule_timeout_uninterruptible(1);
 764         return 0;
 765 }
 766
 767 /*
 768  * Create an RCU-torture statistics message in the specified buffer.
 769  */
 770 static int
 771 rcu_torture_printk(char *page)
 772 {
 773         int cnt = 0;
 774         int cpu;
 775         int i;
 776         long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 777         long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 778
 779         for_each_possible_cpu(cpu) {
 780                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 781                         pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
 782                         batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
 783                 }
 784         }
 785         for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
 786                 if (pipesummary[i] != 0)
 787                         break;
 788         }
 789         cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
 790         cnt += sprintf(&page[cnt],
 791                        "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
 792                        "rtmbe: %d nt: %ld",
 793                        rcu_torture_current,
 794                        rcu_torture_current_version,
 795                        list_empty(&rcu_torture_freelist),
 796                        atomic_read(&n_rcu_torture_alloc),
 797                        atomic_read(&n_rcu_torture_alloc_fail),
 798                        atomic_read(&n_rcu_torture_free),
 799                        atomic_read(&n_rcu_torture_mberror),
 800                        n_rcu_torture_timers);
 801         if (atomic_read(&n_rcu_torture_mberror) != 0)
 802                 cnt += sprintf(&page[cnt], " !!!");
 803         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 804         if (i > 1) {
 805                 cnt += sprintf(&page[cnt], "!!! ");
 806                 atomic_inc(&n_rcu_torture_error);
 807                 WARN_ON_ONCE(1);
 808         }
 809         cnt += sprintf(&page[cnt], "Reader Pipe: ");
 810         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 811                 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
 812         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 813         cnt += sprintf(&page[cnt], "Reader Batch: ");
 814         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 815                 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
 816         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 817         cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
 818         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 819                 cnt += sprintf(&page[cnt], " %d",
 820                                atomic_read(&rcu_torture_wcount[i]));
 821         }
 822         cnt += sprintf(&page[cnt], "\n");
 823         if (cur_ops->stats)
 824                 cnt += cur_ops->stats(&page[cnt]);
 825         return cnt;
 826 }
 827
 828 /*
 829  * Print torture statistics.  Caller must ensure that there is only
 830  * one call to this function at a given time!!!  This is normally
 831  * accomplished by relying on the module system to only have one copy
 832  * of the module loaded, and then by giving the rcu_torture_stats
 833  * kthread full control (or the init/cleanup functions when rcu_torture_stats
 834  * thread is not running).
 835  */
 836 static void
 837 rcu_torture_stats_print(void)
 838 {
 839         int cnt;
 840
 841         cnt = rcu_torture_printk(printk_buf);
 842         printk(KERN_ALERT "%s", printk_buf);
 843 }
 844
 845 /*
 846  * Periodically prints torture statistics, if periodic statistics printing
 847  * was specified via the stat_interval module parameter.
 848  *
 849  * No need to worry about fullstop here, since this one doesn't reference
 850  * volatile state or register callbacks.
 851  */
 852 static int
 853 rcu_torture_stats(void *arg)
 854 {
 855         VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
 856         do {
 857                 schedule_timeout_interruptible(stat_interval * HZ);
 858                 rcu_torture_stats_print();
 859         } while (!kthread_should_stop() && !fullstop);
 860         VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
 861         return 0;
 862 }
 863
 864 static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
 865
 866 /* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
 867  * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
 868  */
 869 static void rcu_torture_shuffle_tasks(void)
 870 {
 871         cpumask_t tmp_mask;
 872         int i;
 873
 874         cpus_setall(tmp_mask);
 875         get_online_cpus();
 876
 877         /* No point in shuffling if there is only one online CPU (ex: UP) */
 878         if (num_online_cpus() == 1) {
 879                 put_online_cpus();
 880                 return;
 881         }
 882
 883         if (rcu_idle_cpu != -1)
 884                 cpu_clear(rcu_idle_cpu, tmp_mask);
 885
 886         set_cpus_allowed_ptr(current, &tmp_mask);
 887
 888         if (reader_tasks) {
 889                 for (i = 0; i < nrealreaders; i++)
 890                         if (reader_tasks[i])
 891                                 set_cpus_allowed_ptr(reader_tasks[i],
 892                                                      &tmp_mask);
 893         }
 894
 895         if (fakewriter_tasks) {
 896                 for (i = 0; i < nfakewriters; i++)
 897                         if (fakewriter_tasks[i])
 898                                 set_cpus_allowed_ptr(fakewriter_tasks[i],
 899                                                      &tmp_mask);
 900         }
 901
 902         if (writer_task)
 903                 set_cpus_allowed_ptr(writer_task, &tmp_mask);
 904
 905         if (stats_task)
 906                 set_cpus_allowed_ptr(stats_task, &tmp_mask);
 907
 908         if (rcu_idle_cpu == -1)
 909                 rcu_idle_cpu = num_online_cpus() - 1;
 910         else
 911                 rcu_idle_cpu--;
 912
 913         put_online_cpus();
 914 }
 915
 916 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 917  * system to become idle at a time and cut off its timer ticks. This is meant
 918  * to test the support for such tickless idle CPU in RCU.
 919  */
 920 static int
 921 rcu_torture_shuffle(void *arg)
 922 {
 923         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
 924         do {
 925                 schedule_timeout_interruptible(shuffle_interval * HZ);
 926                 rcu_torture_shuffle_tasks();
 927         } while (!kthread_should_stop() && !fullstop);
 928         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
 929         return 0;
 930 }
 931
 932 /* Cause the rcutorture test to "stutter", starting and stopping all
 933  * threads periodically.
 934  */
 935 static int
 936 rcu_torture_stutter(void *arg)
 937 {
 938         VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
 939         do {
 940                 schedule_timeout_interruptible(stutter * HZ);
 941                 stutter_pause_test = 1;
 942                 if (!kthread_should_stop() && !fullstop)
 943                         schedule_timeout_interruptible(stutter * HZ);
 944                 stutter_pause_test = 0;
 945         } while (!kthread_should_stop() && !fullstop);
 946         VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
 947         return 0;
 948 }
 949
 950 static inline void
 951 rcu_torture_print_module_parms(char *tag)
 952 {
 953         printk(KERN_ALERT "%s" TORTURE_FLAG
 954                 "--- %s: nreaders=%d nfakewriters=%d "
 955                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
 956                 "shuffle_interval=%d stutter=%d irqreader=%d\n",
 957                 torture_type, tag, nrealreaders, nfakewriters,
 958                 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
 959                 stutter, irqreader);
 960 }
 961
 962 static struct notifier_block rcutorture_nb = {
 963         .notifier_call = rcutorture_shutdown_notify,
 964 };
 965
 966 static void
 967 rcu_torture_cleanup(void)
 968 {
 969         int i;
 970
 971         mutex_lock(&fullstop_mutex);
 972         if (!fullstop) {
 973                 /* If being signaled, let it happen, then exit. */
 974                 mutex_unlock(&fullstop_mutex);
 975                 schedule_timeout_interruptible(10 * HZ);
 976                 if (cur_ops->cb_barrier != NULL)
 977                         cur_ops->cb_barrier();
 978                 return;
 979         }
 980         fullstop = FULLSTOP_CLEANUP;
 981         mutex_unlock(&fullstop_mutex);
 982         unregister_reboot_notifier(&rcutorture_nb);
 983         if (stutter_task) {
 984                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
 985                 kthread_stop(stutter_task);
 986         }
 987         stutter_task = NULL;
 988         if (shuffler_task) {
 989                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
 990                 kthread_stop(shuffler_task);
 991         }
 992         shuffler_task = NULL;
 993
 994         if (writer_task) {
 995                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
 996                 kthread_stop(writer_task);
 997         }
 998         writer_task = NULL;
 999
1000         if (reader_tasks) {
1001                 for (i = 0; i < nrealreaders; i++) {
1002                         if (reader_tasks[i]) {
1003                                 VERBOSE_PRINTK_STRING(
1004                                         "Stopping rcu_torture_reader task");
1005                                 kthread_stop(reader_tasks[i]);
1006                         }
1007                         reader_tasks[i] = NULL;
1008                 }
1009                 kfree(reader_tasks);
1010                 reader_tasks = NULL;
1011         }
1012         rcu_torture_current = NULL;
1013
1014         if (fakewriter_tasks) {
1015                 for (i = 0; i < nfakewriters; i++) {
1016                         if (fakewriter_tasks[i]) {
1017                                 VERBOSE_PRINTK_STRING(
1018                                         "Stopping rcu_torture_fakewriter task");
1019                                 kthread_stop(fakewriter_tasks[i]);
1020                         }
1021                         fakewriter_tasks[i] = NULL;
1022                 }
1023                 kfree(fakewriter_tasks);
1024                 fakewriter_tasks = NULL;
1025         }
1026
1027         if (stats_task) {
1028                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
1029                 kthread_stop(stats_task);
1030         }
1031         stats_task = NULL;
1032
1033         /* Wait for all RCU callbacks to fire.  */
1034
1035         if (cur_ops->cb_barrier != NULL)
1036                 cur_ops->cb_barrier();
1037
1038         rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1039
1040         if (cur_ops->cleanup)
1041                 cur_ops->cleanup();
1042         if (atomic_read(&n_rcu_torture_error))
1043                 rcu_torture_print_module_parms("End of test: FAILURE");
1044         else
1045                 rcu_torture_print_module_parms("End of test: SUCCESS");
1046 }
1047
1048 static int __init
1049 rcu_torture_init(void)
1050 {
1051         int i;
1052         int cpu;
1053         int firsterr = 0;
1054         static struct rcu_torture_ops *torture_ops[] =
1055                 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
1056                   &srcu_ops, &sched_ops, &sched_ops_sync, };
1057
1058         mutex_lock(&fullstop_mutex);
1059
1060         /* Process args and tell the world that the torturer is on the job. */
1061         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1062                 cur_ops = torture_ops[i];
1063                 if (strcmp(torture_type, cur_ops->name) == 0)
1064                         break;
1065         }
1066         if (i == ARRAY_SIZE(torture_ops)) {
1067                 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
1068                        torture_type);
1069                 mutex_unlock(&fullstop_mutex);
1070                 return (-EINVAL);
1071         }
1072         if (cur_ops->init)
1073                 cur_ops->init(); /* no "goto unwind" prior to this point!!! */
1074
1075         if (nreaders >= 0)
1076                 nrealreaders = nreaders;
1077         else
1078                 nrealreaders = 2 * num_online_cpus();
1079         rcu_torture_print_module_parms("Start of test");
1080         fullstop = 0;
1081
1082         /* Set up the freelist. */
1083
1084         INIT_LIST_HEAD(&rcu_torture_freelist);
1085         for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1086                 rcu_tortures[i].rtort_mbtest = 0;
1087                 list_add_tail(&rcu_tortures[i].rtort_free,
1088                               &rcu_torture_freelist);
1089         }
1090
1091         /* Initialize the statistics so that each run gets its own numbers. */
1092
1093         rcu_torture_current = NULL;
1094         rcu_torture_current_version = 0;
1095         atomic_set(&n_rcu_torture_alloc, 0);
1096         atomic_set(&n_rcu_torture_alloc_fail, 0);
1097         atomic_set(&n_rcu_torture_free, 0);
1098         atomic_set(&n_rcu_torture_mberror, 0);
1099         atomic_set(&n_rcu_torture_error, 0);
1100         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1101                 atomic_set(&rcu_torture_wcount[i], 0);
1102         for_each_possible_cpu(cpu) {
1103                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1104                         per_cpu(rcu_torture_count, cpu)[i] = 0;
1105                         per_cpu(rcu_torture_batch, cpu)[i] = 0;
1106                 }
1107         }
1108
1109         /* Start up the kthreads. */
1110
1111         VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
1112         writer_task = kthread_run(rcu_torture_writer, NULL,
1113                                   "rcu_torture_writer");
1114         if (IS_ERR(writer_task)) {
1115                 firsterr = PTR_ERR(writer_task);
1116                 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
1117                 writer_task = NULL;
1118                 goto unwind;
1119         }
1120         fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
1121                                    GFP_KERNEL);
1122         if (fakewriter_tasks == NULL) {
1123                 VERBOSE_PRINTK_ERRSTRING("out of memory");
1124                 firsterr = -ENOMEM;
1125                 goto unwind;
1126         }
1127         for (i = 0; i < nfakewriters; i++) {
1128                 VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
1129                 fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
1130                                                   "rcu_torture_fakewriter");
1131                 if (IS_ERR(fakewriter_tasks[i])) {
1132                         firsterr = PTR_ERR(fakewriter_tasks[i]);
1133                         VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
1134                         fakewriter_tasks[i] = NULL;
1135                         goto unwind;
1136                 }
1137         }
1138         reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
1139                                GFP_KERNEL);
1140         if (reader_tasks == NULL) {
1141                 VERBOSE_PRINTK_ERRSTRING("out of memory");
1142                 firsterr = -ENOMEM;
1143                 goto unwind;
1144         }
1145         for (i = 0; i < nrealreaders; i++) {
1146                 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
1147                 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
1148                                               "rcu_torture_reader");
1149                 if (IS_ERR(reader_tasks[i])) {
1150                         firsterr = PTR_ERR(reader_tasks[i]);
1151                         VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
1152                         reader_tasks[i] = NULL;
1153                         goto unwind;
1154                 }
1155         }
1156         if (stat_interval > 0) {
1157                 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
1158                 stats_task = kthread_run(rcu_torture_stats, NULL,
1159                                         "rcu_torture_stats");
1160                 if (IS_ERR(stats_task)) {
1161                         firsterr = PTR_ERR(stats_task);
1162                         VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
1163                         stats_task = NULL;
1164                         goto unwind;
1165                 }
1166         }
1167         if (test_no_idle_hz) {
1168                 rcu_idle_cpu = num_online_cpus() - 1;
1169                 /* Create the shuffler thread */
1170                 shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
1171                                           "rcu_torture_shuffle");
1172                 if (IS_ERR(shuffler_task)) {
1173                         firsterr = PTR_ERR(shuffler_task);
1174                         VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
1175                         shuffler_task = NULL;
1176                         goto unwind;
1177                 }
1178         }
1179         if (stutter < 0)
1180                 stutter = 0;
1181         if (stutter) {
1182                 /* Create the stutter thread */
1183                 stutter_task = kthread_run(rcu_torture_stutter, NULL,
1184                                           "rcu_torture_stutter");
1185                 if (IS_ERR(stutter_task)) {
1186                         firsterr = PTR_ERR(stutter_task);
1187                         VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
1188                         stutter_task = NULL;
1189                         goto unwind;
1190                 }
1191         }
1192         register_reboot_notifier(&rcutorture_nb);
1193         mutex_unlock(&fullstop_mutex);
1194         return 0;
1195
1196 unwind:
1197         mutex_unlock(&fullstop_mutex);
1198         rcu_torture_cleanup();
1199         return firsterr;
1200 }
1201
1202 module_init(rcu_torture_init);
1203 module_exit(rcu_torture_cleanup);