kernel/rcu/refscale.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 //
   3 // Scalability test comparing RCU vs other mechanisms
   4 // for acquiring references on objects.
   5 //
   6 // Copyright (C) Google, 2020.
   7 //
   8 // Author: Joel Fernandes <joel@joelfernandes.org>
   9
  10 #define pr_fmt(fmt) fmt
  11
  12 #include <linux/atomic.h>
  13 #include <linux/bitops.h>
  14 #include <linux/completion.h>
  15 #include <linux/cpu.h>
  16 #include <linux/delay.h>
  17 #include <linux/err.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/kthread.h>
  21 #include <linux/kernel.h>
  22 #include <linux/mm.h>
  23 #include <linux/module.h>
  24 #include <linux/moduleparam.h>
  25 #include <linux/notifier.h>
  26 #include <linux/percpu.h>
  27 #include <linux/rcupdate.h>
  28 #include <linux/rcupdate_trace.h>
  29 #include <linux/reboot.h>
  30 #include <linux/sched.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/smp.h>
  33 #include <linux/stat.h>
  34 #include <linux/srcu.h>
  35 #include <linux/slab.h>
  36 #include <linux/torture.h>
  37 #include <linux/types.h>
  38
  39 #include "rcu.h"
  40
  41 #define SCALE_FLAG "-ref-scale: "
  42
  43 #define SCALEOUT(s, x...) \
  44         pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
  45
  46 #define VERBOSE_SCALEOUT(s, x...) \
  47         do { \
  48                 if (verbose) \
  49                         pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
  50         } while (0)
  51
  52 static atomic_t verbose_batch_ctr;
  53
  54 #define VERBOSE_SCALEOUT_BATCH(s, x...)                                                 \
  55 do {                                                                                    \
  56         if (verbose &&                                                                  \
  57             (verbose_batched <= 0 ||                                                    \
  58              !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) {             \
  59                 schedule_timeout_uninterruptible(1);                                    \
  60                 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x);                     \
  61         }                                                                               \
  62 } while (0)
  63
  64 #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
  65
  66 MODULE_LICENSE("GPL");
  67 MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
  68
  69 static char *scale_type = "rcu";
  70 module_param(scale_type, charp, 0444);
  71 MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
  72
  73 torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
  74 torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
  75
  76 // Wait until there are multiple CPUs before starting test.
  77 torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
  78               "Holdoff time before test start (s)");
  79 // Number of typesafe_lookup structures, that is, the degree of concurrency.
  80 torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
  81 // Number of loops per experiment, all readers execute operations concurrently.
  82 torture_param(long, loops, 10000, "Number of loops per experiment.");
  83 // Number of readers, with -1 defaulting to about 75% of the CPUs.
  84 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
  85 // Number of runs.
  86 torture_param(int, nruns, 30, "Number of experiments to run.");
  87 // Reader delay in nanoseconds, 0 for no delay.
  88 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
  89
  90 #ifdef MODULE
  91 # define REFSCALE_SHUTDOWN 0
  92 #else
  93 # define REFSCALE_SHUTDOWN 1
  94 #endif
  95
  96 torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
  97               "Shutdown at end of scalability tests.");
  98
  99 struct reader_task {
 100         struct task_struct *task;
 101         int start_reader;
 102         wait_queue_head_t wq;
 103         u64 last_duration_ns;
 104 };
 105
 106 static struct task_struct *shutdown_task;
 107 static wait_queue_head_t shutdown_wq;
 108
 109 static struct task_struct *main_task;
 110 static wait_queue_head_t main_wq;
 111 static int shutdown_start;
 112
 113 static struct reader_task *reader_tasks;
 114
 115 // Number of readers that are part of the current experiment.
 116 static atomic_t nreaders_exp;
 117
 118 // Use to wait for all threads to start.
 119 static atomic_t n_init;
 120 static atomic_t n_started;
 121 static atomic_t n_warmedup;
 122 static atomic_t n_cooleddown;
 123
 124 // Track which experiment is currently running.
 125 static int exp_idx;
 126
 127 // Operations vector for selecting different types of tests.
 128 struct ref_scale_ops {
 129         bool (*init)(void);
 130         void (*cleanup)(void);
 131         void (*readsection)(const int nloops);
 132         void (*delaysection)(const int nloops, const int udl, const int ndl);
 133         const char *name;
 134 };
 135
 136 static struct ref_scale_ops *cur_ops;
 137
 138 static void un_delay(const int udl, const int ndl)
 139 {
 140         if (udl)
 141                 udelay(udl);
 142         if (ndl)
 143                 ndelay(ndl);
 144 }
 145
 146 static void ref_rcu_read_section(const int nloops)
 147 {
 148         int i;
 149
 150         for (i = nloops; i >= 0; i--) {
 151                 rcu_read_lock();
 152                 rcu_read_unlock();
 153         }
 154 }
 155
 156 static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
 157 {
 158         int i;
 159
 160         for (i = nloops; i >= 0; i--) {
 161                 rcu_read_lock();
 162                 un_delay(udl, ndl);
 163                 rcu_read_unlock();
 164         }
 165 }
 166
 167 static bool rcu_sync_scale_init(void)
 168 {
 169         return true;
 170 }
 171
 172 static struct ref_scale_ops rcu_ops = {
 173         .init           = rcu_sync_scale_init,
 174         .readsection    = ref_rcu_read_section,
 175         .delaysection   = ref_rcu_delay_section,
 176         .name           = "rcu"
 177 };
 178
 179 // Definitions for SRCU ref scale testing.
 180 DEFINE_STATIC_SRCU(srcu_refctl_scale);
 181 static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
 182
 183 static void srcu_ref_scale_read_section(const int nloops)
 184 {
 185         int i;
 186         int idx;
 187
 188         for (i = nloops; i >= 0; i--) {
 189                 idx = srcu_read_lock(srcu_ctlp);
 190                 srcu_read_unlock(srcu_ctlp, idx);
 191         }
 192 }
 193
 194 static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
 195 {
 196         int i;
 197         int idx;
 198
 199         for (i = nloops; i >= 0; i--) {
 200                 idx = srcu_read_lock(srcu_ctlp);
 201                 un_delay(udl, ndl);
 202                 srcu_read_unlock(srcu_ctlp, idx);
 203         }
 204 }
 205
 206 static struct ref_scale_ops srcu_ops = {
 207         .init           = rcu_sync_scale_init,
 208         .readsection    = srcu_ref_scale_read_section,
 209         .delaysection   = srcu_ref_scale_delay_section,
 210         .name           = "srcu"
 211 };
 212
 213 #ifdef CONFIG_TASKS_RCU
 214
 215 // Definitions for RCU Tasks ref scale testing: Empty read markers.
 216 // These definitions also work for RCU Rude readers.
 217 static void rcu_tasks_ref_scale_read_section(const int nloops)
 218 {
 219         int i;
 220
 221         for (i = nloops; i >= 0; i--)
 222                 continue;
 223 }
 224
 225 static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
 226 {
 227         int i;
 228
 229         for (i = nloops; i >= 0; i--)
 230                 un_delay(udl, ndl);
 231 }
 232
 233 static struct ref_scale_ops rcu_tasks_ops = {
 234         .init           = rcu_sync_scale_init,
 235         .readsection    = rcu_tasks_ref_scale_read_section,
 236         .delaysection   = rcu_tasks_ref_scale_delay_section,
 237         .name           = "rcu-tasks"
 238 };
 239
 240 #define RCU_TASKS_OPS &rcu_tasks_ops,
 241
 242 #else // #ifdef CONFIG_TASKS_RCU
 243
 244 #define RCU_TASKS_OPS
 245
 246 #endif // #else // #ifdef CONFIG_TASKS_RCU
 247
 248 #ifdef CONFIG_TASKS_TRACE_RCU
 249
 250 // Definitions for RCU Tasks Trace ref scale testing.
 251 static void rcu_trace_ref_scale_read_section(const int nloops)
 252 {
 253         int i;
 254
 255         for (i = nloops; i >= 0; i--) {
 256                 rcu_read_lock_trace();
 257                 rcu_read_unlock_trace();
 258         }
 259 }
 260
 261 static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
 262 {
 263         int i;
 264
 265         for (i = nloops; i >= 0; i--) {
 266                 rcu_read_lock_trace();
 267                 un_delay(udl, ndl);
 268                 rcu_read_unlock_trace();
 269         }
 270 }
 271
 272 static struct ref_scale_ops rcu_trace_ops = {
 273         .init           = rcu_sync_scale_init,
 274         .readsection    = rcu_trace_ref_scale_read_section,
 275         .delaysection   = rcu_trace_ref_scale_delay_section,
 276         .name           = "rcu-trace"
 277 };
 278
 279 #define RCU_TRACE_OPS &rcu_trace_ops,
 280
 281 #else // #ifdef CONFIG_TASKS_TRACE_RCU
 282
 283 #define RCU_TRACE_OPS
 284
 285 #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU
 286
 287 // Definitions for reference count
 288 static atomic_t refcnt;
 289
 290 static void ref_refcnt_section(const int nloops)
 291 {
 292         int i;
 293
 294         for (i = nloops; i >= 0; i--) {
 295                 atomic_inc(&refcnt);
 296                 atomic_dec(&refcnt);
 297         }
 298 }
 299
 300 static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
 301 {
 302         int i;
 303
 304         for (i = nloops; i >= 0; i--) {
 305                 atomic_inc(&refcnt);
 306                 un_delay(udl, ndl);
 307                 atomic_dec(&refcnt);
 308         }
 309 }
 310
 311 static struct ref_scale_ops refcnt_ops = {
 312         .init           = rcu_sync_scale_init,
 313         .readsection    = ref_refcnt_section,
 314         .delaysection   = ref_refcnt_delay_section,
 315         .name           = "refcnt"
 316 };
 317
 318 // Definitions for rwlock
 319 static rwlock_t test_rwlock;
 320
 321 static bool ref_rwlock_init(void)
 322 {
 323         rwlock_init(&test_rwlock);
 324         return true;
 325 }
 326
 327 static void ref_rwlock_section(const int nloops)
 328 {
 329         int i;
 330
 331         for (i = nloops; i >= 0; i--) {
 332                 read_lock(&test_rwlock);
 333                 read_unlock(&test_rwlock);
 334         }
 335 }
 336
 337 static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
 338 {
 339         int i;
 340
 341         for (i = nloops; i >= 0; i--) {
 342                 read_lock(&test_rwlock);
 343                 un_delay(udl, ndl);
 344                 read_unlock(&test_rwlock);
 345         }
 346 }
 347
 348 static struct ref_scale_ops rwlock_ops = {
 349         .init           = ref_rwlock_init,
 350         .readsection    = ref_rwlock_section,
 351         .delaysection   = ref_rwlock_delay_section,
 352         .name           = "rwlock"
 353 };
 354
 355 // Definitions for rwsem
 356 static struct rw_semaphore test_rwsem;
 357
 358 static bool ref_rwsem_init(void)
 359 {
 360         init_rwsem(&test_rwsem);
 361         return true;
 362 }
 363
 364 static void ref_rwsem_section(const int nloops)
 365 {
 366         int i;
 367
 368         for (i = nloops; i >= 0; i--) {
 369                 down_read(&test_rwsem);
 370                 up_read(&test_rwsem);
 371         }
 372 }
 373
 374 static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
 375 {
 376         int i;
 377
 378         for (i = nloops; i >= 0; i--) {
 379                 down_read(&test_rwsem);
 380                 un_delay(udl, ndl);
 381                 up_read(&test_rwsem);
 382         }
 383 }
 384
 385 static struct ref_scale_ops rwsem_ops = {
 386         .init           = ref_rwsem_init,
 387         .readsection    = ref_rwsem_section,
 388         .delaysection   = ref_rwsem_delay_section,
 389         .name           = "rwsem"
 390 };
 391
 392 // Definitions for global spinlock
 393 static DEFINE_RAW_SPINLOCK(test_lock);
 394
 395 static void ref_lock_section(const int nloops)
 396 {
 397         int i;
 398
 399         preempt_disable();
 400         for (i = nloops; i >= 0; i--) {
 401                 raw_spin_lock(&test_lock);
 402                 raw_spin_unlock(&test_lock);
 403         }
 404         preempt_enable();
 405 }
 406
 407 static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
 408 {
 409         int i;
 410
 411         preempt_disable();
 412         for (i = nloops; i >= 0; i--) {
 413                 raw_spin_lock(&test_lock);
 414                 un_delay(udl, ndl);
 415                 raw_spin_unlock(&test_lock);
 416         }
 417         preempt_enable();
 418 }
 419
 420 static struct ref_scale_ops lock_ops = {
 421         .readsection    = ref_lock_section,
 422         .delaysection   = ref_lock_delay_section,
 423         .name           = "lock"
 424 };
 425
 426 // Definitions for global irq-save spinlock
 427
 428 static void ref_lock_irq_section(const int nloops)
 429 {
 430         unsigned long flags;
 431         int i;
 432
 433         preempt_disable();
 434         for (i = nloops; i >= 0; i--) {
 435                 raw_spin_lock_irqsave(&test_lock, flags);
 436                 raw_spin_unlock_irqrestore(&test_lock, flags);
 437         }
 438         preempt_enable();
 439 }
 440
 441 static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
 442 {
 443         unsigned long flags;
 444         int i;
 445
 446         preempt_disable();
 447         for (i = nloops; i >= 0; i--) {
 448                 raw_spin_lock_irqsave(&test_lock, flags);
 449                 un_delay(udl, ndl);
 450                 raw_spin_unlock_irqrestore(&test_lock, flags);
 451         }
 452         preempt_enable();
 453 }
 454
 455 static struct ref_scale_ops lock_irq_ops = {
 456         .readsection    = ref_lock_irq_section,
 457         .delaysection   = ref_lock_irq_delay_section,
 458         .name           = "lock-irq"
 459 };
 460
 461 // Definitions acquire-release.
 462 static DEFINE_PER_CPU(unsigned long, test_acqrel);
 463
 464 static void ref_acqrel_section(const int nloops)
 465 {
 466         unsigned long x;
 467         int i;
 468
 469         preempt_disable();
 470         for (i = nloops; i >= 0; i--) {
 471                 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
 472                 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
 473         }
 474         preempt_enable();
 475 }
 476
 477 static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
 478 {
 479         unsigned long x;
 480         int i;
 481
 482         preempt_disable();
 483         for (i = nloops; i >= 0; i--) {
 484                 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
 485                 un_delay(udl, ndl);
 486                 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
 487         }
 488         preempt_enable();
 489 }
 490
 491 static struct ref_scale_ops acqrel_ops = {
 492         .readsection    = ref_acqrel_section,
 493         .delaysection   = ref_acqrel_delay_section,
 494         .name           = "acqrel"
 495 };
 496
 497 static volatile u64 stopopts;
 498
 499 static void ref_clock_section(const int nloops)
 500 {
 501         u64 x = 0;
 502         int i;
 503
 504         preempt_disable();
 505         for (i = nloops; i >= 0; i--)
 506                 x += ktime_get_real_fast_ns();
 507         preempt_enable();
 508         stopopts = x;
 509 }
 510
 511 static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
 512 {
 513         u64 x = 0;
 514         int i;
 515
 516         preempt_disable();
 517         for (i = nloops; i >= 0; i--) {
 518                 x += ktime_get_real_fast_ns();
 519                 un_delay(udl, ndl);
 520         }
 521         preempt_enable();
 522         stopopts = x;
 523 }
 524
 525 static struct ref_scale_ops clock_ops = {
 526         .readsection    = ref_clock_section,
 527         .delaysection   = ref_clock_delay_section,
 528         .name           = "clock"
 529 };
 530
 531 static void ref_jiffies_section(const int nloops)
 532 {
 533         u64 x = 0;
 534         int i;
 535
 536         preempt_disable();
 537         for (i = nloops; i >= 0; i--)
 538                 x += jiffies;
 539         preempt_enable();
 540         stopopts = x;
 541 }
 542
 543 static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl)
 544 {
 545         u64 x = 0;
 546         int i;
 547
 548         preempt_disable();
 549         for (i = nloops; i >= 0; i--) {
 550                 x += jiffies;
 551                 un_delay(udl, ndl);
 552         }
 553         preempt_enable();
 554         stopopts = x;
 555 }
 556
 557 static struct ref_scale_ops jiffies_ops = {
 558         .readsection    = ref_jiffies_section,
 559         .delaysection   = ref_jiffies_delay_section,
 560         .name           = "jiffies"
 561 };
 562
 563 ////////////////////////////////////////////////////////////////////////
 564 //
 565 // Methods leveraging SLAB_TYPESAFE_BY_RCU.
 566 //
 567
 568 // Item to look up in a typesafe manner.  Array of pointers to these.
 569 struct refscale_typesafe {
 570         atomic_t rts_refctr;  // Used by all flavors
 571         spinlock_t rts_lock;
 572         seqlock_t rts_seqlock;
 573         unsigned int a;
 574         unsigned int b;
 575 };
 576
 577 static struct kmem_cache *typesafe_kmem_cachep;
 578 static struct refscale_typesafe **rtsarray;
 579 static long rtsarray_size;
 580 static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand);
 581 static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start);
 582 static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start);
 583
 584 // Conditionally acquire an explicit in-structure reference count.
 585 static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
 586 {
 587         return atomic_inc_not_zero(&rtsp->rts_refctr);
 588 }
 589
 590 // Unconditionally release an explicit in-structure reference count.
 591 static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start)
 592 {
 593         if (!atomic_dec_return(&rtsp->rts_refctr)) {
 594                 WRITE_ONCE(rtsp->a, rtsp->a + 1);
 595                 kmem_cache_free(typesafe_kmem_cachep, rtsp);
 596         }
 597         return true;
 598 }
 599
 600 // Unconditionally acquire an explicit in-structure spinlock.
 601 static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
 602 {
 603         spin_lock(&rtsp->rts_lock);
 604         return true;
 605 }
 606
 607 // Unconditionally release an explicit in-structure spinlock.
 608 static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start)
 609 {
 610         spin_unlock(&rtsp->rts_lock);
 611         return true;
 612 }
 613
 614 // Unconditionally acquire an explicit in-structure sequence lock.
 615 static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
 616 {
 617         *start = read_seqbegin(&rtsp->rts_seqlock);
 618         return true;
 619 }
 620
 621 // Conditionally release an explicit in-structure sequence lock.  Return
 622 // true if this release was successful, that is, if no retry is required.
 623 static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start)
 624 {
 625         return !read_seqretry(&rtsp->rts_seqlock, start);
 626 }
 627
 628 // Do a read-side critical section with the specified delay in
 629 // microseconds and nanoseconds inserted so as to increase probability
 630 // of failure.
 631 static void typesafe_delay_section(const int nloops, const int udl, const int ndl)
 632 {
 633         unsigned int a;
 634         unsigned int b;
 635         int i;
 636         long idx;
 637         struct refscale_typesafe *rtsp;
 638         unsigned int start;
 639
 640         for (i = nloops; i >= 0; i--) {
 641                 preempt_disable();
 642                 idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size;
 643                 preempt_enable();
 644 retry:
 645                 rcu_read_lock();
 646                 rtsp = rcu_dereference(rtsarray[idx]);
 647                 a = READ_ONCE(rtsp->a);
 648                 if (!rts_acquire(rtsp, &start)) {
 649                         rcu_read_unlock();
 650                         goto retry;
 651                 }
 652                 if (a != READ_ONCE(rtsp->a)) {
 653                         (void)rts_release(rtsp, start);
 654                         rcu_read_unlock();
 655                         goto retry;
 656                 }
 657                 un_delay(udl, ndl);
 658                 // Remember, seqlock read-side release can fail.
 659                 if (!rts_release(rtsp, start)) {
 660                         rcu_read_unlock();
 661                         goto retry;
 662                 }
 663                 b = READ_ONCE(rtsp->a);
 664                 WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);
 665                 b = rtsp->b;
 666                 rcu_read_unlock();
 667                 WARN_ON_ONCE(a * a != b);
 668         }
 669 }
 670
 671 // Because the acquisition and release methods are expensive, there
 672 // is no point in optimizing away the un_delay() function's two checks.
 673 // Thus simply define typesafe_read_section() as a simple wrapper around
 674 // typesafe_delay_section().
 675 static void typesafe_read_section(const int nloops)
 676 {
 677         typesafe_delay_section(nloops, 0, 0);
 678 }
 679
 680 // Allocate and initialize one refscale_typesafe structure.
 681 static struct refscale_typesafe *typesafe_alloc_one(void)
 682 {
 683         struct refscale_typesafe *rtsp;
 684
 685         rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL);
 686         if (!rtsp)
 687                 return NULL;
 688         atomic_set(&rtsp->rts_refctr, 1);
 689         WRITE_ONCE(rtsp->a, rtsp->a + 1);
 690         WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);
 691         return rtsp;
 692 }
 693
 694 // Slab-allocator constructor for refscale_typesafe structures created
 695 // out of a new slab of system memory.
 696 static void refscale_typesafe_ctor(void *rtsp_in)
 697 {
 698         struct refscale_typesafe *rtsp = rtsp_in;
 699
 700         spin_lock_init(&rtsp->rts_lock);
 701         seqlock_init(&rtsp->rts_seqlock);
 702         preempt_disable();
 703         rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));
 704         preempt_enable();
 705 }
 706
 707 static struct ref_scale_ops typesafe_ref_ops;
 708 static struct ref_scale_ops typesafe_lock_ops;
 709 static struct ref_scale_ops typesafe_seqlock_ops;
 710
 711 // Initialize for a typesafe test.
 712 static bool typesafe_init(void)
 713 {
 714         long idx;
 715         long si = lookup_instances;
 716
 717         typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe",
 718                                                  sizeof(struct refscale_typesafe), sizeof(void *),
 719                                                  SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor);
 720         if (!typesafe_kmem_cachep)
 721                 return false;
 722         if (si < 0)
 723                 si = -si * nr_cpu_ids;
 724         else if (si == 0)
 725                 si = nr_cpu_ids;
 726         rtsarray_size = si;
 727         rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL);
 728         if (!rtsarray)
 729                 return false;
 730         for (idx = 0; idx < rtsarray_size; idx++) {
 731                 rtsarray[idx] = typesafe_alloc_one();
 732                 if (!rtsarray[idx])
 733                         return false;
 734         }
 735         if (cur_ops == &typesafe_ref_ops) {
 736                 rts_acquire = typesafe_ref_acquire;
 737                 rts_release = typesafe_ref_release;
 738         } else if (cur_ops == &typesafe_lock_ops) {
 739                 rts_acquire = typesafe_lock_acquire;
 740                 rts_release = typesafe_lock_release;
 741         } else if (cur_ops == &typesafe_seqlock_ops) {
 742                 rts_acquire = typesafe_seqlock_acquire;
 743                 rts_release = typesafe_seqlock_release;
 744         } else {
 745                 WARN_ON_ONCE(1);
 746                 return false;
 747         }
 748         return true;
 749 }
 750
 751 // Clean up after a typesafe test.
 752 static void typesafe_cleanup(void)
 753 {
 754         long idx;
 755
 756         if (rtsarray) {
 757                 for (idx = 0; idx < rtsarray_size; idx++)
 758                         kmem_cache_free(typesafe_kmem_cachep, rtsarray[idx]);
 759                 kfree(rtsarray);
 760                 rtsarray = NULL;
 761                 rtsarray_size = 0;
 762         }
 763         kmem_cache_destroy(typesafe_kmem_cachep);
 764         typesafe_kmem_cachep = NULL;
 765         rts_acquire = NULL;
 766         rts_release = NULL;
 767 }
 768
 769 // The typesafe_init() function distinguishes these structures by address.
 770 static struct ref_scale_ops typesafe_ref_ops = {
 771         .init           = typesafe_init,
 772         .cleanup        = typesafe_cleanup,
 773         .readsection    = typesafe_read_section,
 774         .delaysection   = typesafe_delay_section,
 775         .name           = "typesafe_ref"
 776 };
 777
 778 static struct ref_scale_ops typesafe_lock_ops = {
 779         .init           = typesafe_init,
 780         .cleanup        = typesafe_cleanup,
 781         .readsection    = typesafe_read_section,
 782         .delaysection   = typesafe_delay_section,
 783         .name           = "typesafe_lock"
 784 };
 785
 786 static struct ref_scale_ops typesafe_seqlock_ops = {
 787         .init           = typesafe_init,
 788         .cleanup        = typesafe_cleanup,
 789         .readsection    = typesafe_read_section,
 790         .delaysection   = typesafe_delay_section,
 791         .name           = "typesafe_seqlock"
 792 };
 793
 794 static void rcu_scale_one_reader(void)
 795 {
 796         if (readdelay <= 0)
 797                 cur_ops->readsection(loops);
 798         else
 799                 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
 800 }
 801
 802 // Reader kthread.  Repeatedly does empty RCU read-side
 803 // critical section, minimizing update-side interference.
 804 static int
 805 ref_scale_reader(void *arg)
 806 {
 807         unsigned long flags;
 808         long me = (long)arg;
 809         struct reader_task *rt = &(reader_tasks[me]);
 810         u64 start;
 811         s64 duration;
 812
 813         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
 814         WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
 815         set_user_nice(current, MAX_NICE);
 816         atomic_inc(&n_init);
 817         if (holdoff)
 818                 schedule_timeout_interruptible(holdoff * HZ);
 819 repeat:
 820         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
 821
 822         // Wait for signal that this reader can start.
 823         wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
 824                            torture_must_stop());
 825
 826         if (torture_must_stop())
 827                 goto end;
 828
 829         // Make sure that the CPU is affinitized appropriately during testing.
 830         WARN_ON_ONCE(raw_smp_processor_id() != me);
 831
 832         WRITE_ONCE(rt->start_reader, 0);
 833         if (!atomic_dec_return(&n_started))
 834                 while (atomic_read_acquire(&n_started))
 835                         cpu_relax();
 836
 837         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
 838
 839
 840         // To reduce noise, do an initial cache-warming invocation, check
 841         // in, and then keep warming until everyone has checked in.
 842         rcu_scale_one_reader();
 843         if (!atomic_dec_return(&n_warmedup))
 844                 while (atomic_read_acquire(&n_warmedup))
 845                         rcu_scale_one_reader();
 846         // Also keep interrupts disabled.  This also has the effect
 847         // of preventing entries into slow path for rcu_read_unlock().
 848         local_irq_save(flags);
 849         start = ktime_get_mono_fast_ns();
 850
 851         rcu_scale_one_reader();
 852
 853         duration = ktime_get_mono_fast_ns() - start;
 854         local_irq_restore(flags);
 855
 856         rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
 857         // To reduce runtime-skew noise, do maintain-load invocations until
 858         // everyone is done.
 859         if (!atomic_dec_return(&n_cooleddown))
 860                 while (atomic_read_acquire(&n_cooleddown))
 861                         rcu_scale_one_reader();
 862
 863         if (atomic_dec_and_test(&nreaders_exp))
 864                 wake_up(&main_wq);
 865
 866         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
 867                                 me, exp_idx, atomic_read(&nreaders_exp));
 868
 869         if (!torture_must_stop())
 870                 goto repeat;
 871 end:
 872         torture_kthread_stopping("ref_scale_reader");
 873         return 0;
 874 }
 875
 876 static void reset_readers(void)
 877 {
 878         int i;
 879         struct reader_task *rt;
 880
 881         for (i = 0; i < nreaders; i++) {
 882                 rt = &(reader_tasks[i]);
 883
 884                 rt->last_duration_ns = 0;
 885         }
 886 }
 887
 888 // Print the results of each reader and return the sum of all their durations.
 889 static u64 process_durations(int n)
 890 {
 891         int i;
 892         struct reader_task *rt;
 893         char buf1[64];
 894         char *buf;
 895         u64 sum = 0;
 896
 897         buf = kmalloc(800 + 64, GFP_KERNEL);
 898         if (!buf)
 899                 return 0;
 900         buf[0] = 0;
 901         sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
 902                 exp_idx);
 903
 904         for (i = 0; i < n && !torture_must_stop(); i++) {
 905                 rt = &(reader_tasks[i]);
 906                 sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);
 907
 908                 if (i % 5 == 0)
 909                         strcat(buf, "\n");
 910                 if (strlen(buf) >= 800) {
 911                         pr_alert("%s", buf);
 912                         buf[0] = 0;
 913                 }
 914                 strcat(buf, buf1);
 915
 916                 sum += rt->last_duration_ns;
 917         }
 918         pr_alert("%s\n", buf);
 919
 920         kfree(buf);
 921         return sum;
 922 }
 923
 924 // The main_func is the main orchestrator, it performs a bunch of
 925 // experiments.  For every experiment, it orders all the readers
 926 // involved to start and waits for them to finish the experiment. It
 927 // then reads their timestamps and starts the next experiment. Each
 928 // experiment progresses from 1 concurrent reader to N of them at which
 929 // point all the timestamps are printed.
 930 static int main_func(void *arg)
 931 {
 932         int exp, r;
 933         char buf1[64];
 934         char *buf;
 935         u64 *result_avg;
 936
 937         set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
 938         set_user_nice(current, MAX_NICE);
 939
 940         VERBOSE_SCALEOUT("main_func task started");
 941         result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
 942         buf = kzalloc(800 + 64, GFP_KERNEL);
 943         if (!result_avg || !buf) {
 944                 SCALEOUT_ERRSTRING("out of memory");
 945                 goto oom_exit;
 946         }
 947         if (holdoff)
 948                 schedule_timeout_interruptible(holdoff * HZ);
 949
 950         // Wait for all threads to start.
 951         atomic_inc(&n_init);
 952         while (atomic_read(&n_init) < nreaders + 1)
 953                 schedule_timeout_uninterruptible(1);
 954
 955         // Start exp readers up per experiment
 956         for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
 957                 if (torture_must_stop())
 958                         goto end;
 959
 960                 reset_readers();
 961                 atomic_set(&nreaders_exp, nreaders);
 962                 atomic_set(&n_started, nreaders);
 963                 atomic_set(&n_warmedup, nreaders);
 964                 atomic_set(&n_cooleddown, nreaders);
 965
 966                 exp_idx = exp;
 967
 968                 for (r = 0; r < nreaders; r++) {
 969                         smp_store_release(&reader_tasks[r].start_reader, 1);
 970                         wake_up(&reader_tasks[r].wq);
 971                 }
 972
 973                 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
 974                                 nreaders);
 975
 976                 wait_event(main_wq,
 977                            !atomic_read(&nreaders_exp) || torture_must_stop());
 978
 979                 VERBOSE_SCALEOUT("main_func: experiment ended");
 980
 981                 if (torture_must_stop())
 982                         goto end;
 983
 984                 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
 985         }
 986
 987         // Print the average of all experiments
 988         SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
 989
 990         pr_alert("Runs\tTime(ns)\n");
 991         for (exp = 0; exp < nruns; exp++) {
 992                 u64 avg;
 993                 u32 rem;
 994
 995                 avg = div_u64_rem(result_avg[exp], 1000, &rem);
 996                 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
 997                 strcat(buf, buf1);
 998                 if (strlen(buf) >= 800) {
 999                         pr_alert("%s", buf);
1000                         buf[0] = 0;
1001                 }
1002         }
1003
1004         pr_alert("%s", buf);
1005
1006 oom_exit:
1007         // This will shutdown everything including us.
1008         if (shutdown) {
1009                 shutdown_start = 1;
1010                 wake_up(&shutdown_wq);
1011         }
1012
1013         // Wait for torture to stop us
1014         while (!torture_must_stop())
1015                 schedule_timeout_uninterruptible(1);
1016
1017 end:
1018         torture_kthread_stopping("main_func");
1019         kfree(result_avg);
1020         kfree(buf);
1021         return 0;
1022 }
1023
1024 static void
1025 ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
1026 {
1027         pr_alert("%s" SCALE_FLAG
1028                  "--- %s:  verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
1029                  verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay);
1030 }
1031
1032 static void
1033 ref_scale_cleanup(void)
1034 {
1035         int i;
1036
1037         if (torture_cleanup_begin())
1038                 return;
1039
1040         if (!cur_ops) {
1041                 torture_cleanup_end();
1042                 return;
1043         }
1044
1045         if (reader_tasks) {
1046                 for (i = 0; i < nreaders; i++)
1047                         torture_stop_kthread("ref_scale_reader",
1048                                              reader_tasks[i].task);
1049         }
1050         kfree(reader_tasks);
1051
1052         torture_stop_kthread("main_task", main_task);
1053         kfree(main_task);
1054
1055         // Do scale-type-specific cleanup operations.
1056         if (cur_ops->cleanup != NULL)
1057                 cur_ops->cleanup();
1058
1059         torture_cleanup_end();
1060 }
1061
1062 // Shutdown kthread.  Just waits to be awakened, then shuts down system.
1063 static int
1064 ref_scale_shutdown(void *arg)
1065 {
1066         wait_event_idle(shutdown_wq, shutdown_start);
1067
1068         smp_mb(); // Wake before output.
1069         ref_scale_cleanup();
1070         kernel_power_off();
1071
1072         return -EINVAL;
1073 }
1074
1075 static int __init
1076 ref_scale_init(void)
1077 {
1078         long i;
1079         int firsterr = 0;
1080         static struct ref_scale_ops *scale_ops[] = {
1081                 &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops,
1082                 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops,
1083                 &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
1084         };
1085
1086         if (!torture_init_begin(scale_type, verbose))
1087                 return -EBUSY;
1088
1089         for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
1090                 cur_ops = scale_ops[i];
1091                 if (strcmp(scale_type, cur_ops->name) == 0)
1092                         break;
1093         }
1094         if (i == ARRAY_SIZE(scale_ops)) {
1095                 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
1096                 pr_alert("rcu-scale types:");
1097                 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
1098                         pr_cont(" %s", scale_ops[i]->name);
1099                 pr_cont("\n");
1100                 firsterr = -EINVAL;
1101                 cur_ops = NULL;
1102                 goto unwind;
1103         }
1104         if (cur_ops->init)
1105                 if (!cur_ops->init()) {
1106                         firsterr = -EUCLEAN;
1107                         goto unwind;
1108                 }
1109
1110         ref_scale_print_module_parms(cur_ops, "Start of test");
1111
1112         // Shutdown task
1113         if (shutdown) {
1114                 init_waitqueue_head(&shutdown_wq);
1115                 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
1116                                                   shutdown_task);
1117                 if (torture_init_error(firsterr))
1118                         goto unwind;
1119                 schedule_timeout_uninterruptible(1);
1120         }
1121
1122         // Reader tasks (default to ~75% of online CPUs).
1123         if (nreaders < 0)
1124                 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
1125         if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
1126                 loops = 1;
1127         if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
1128                 nreaders = 1;
1129         if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
1130                 nruns = 1;
1131         reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
1132                                GFP_KERNEL);
1133         if (!reader_tasks) {
1134                 SCALEOUT_ERRSTRING("out of memory");
1135                 firsterr = -ENOMEM;
1136                 goto unwind;
1137         }
1138
1139         VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
1140
1141         for (i = 0; i < nreaders; i++) {
1142                 init_waitqueue_head(&reader_tasks[i].wq);
1143                 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
1144                                                   reader_tasks[i].task);
1145                 if (torture_init_error(firsterr))
1146                         goto unwind;
1147         }
1148
1149         // Main Task
1150         init_waitqueue_head(&main_wq);
1151         firsterr = torture_create_kthread(main_func, NULL, main_task);
1152         if (torture_init_error(firsterr))
1153                 goto unwind;
1154
1155         torture_init_end();
1156         return 0;
1157
1158 unwind:
1159         torture_init_end();
1160         ref_scale_cleanup();
1161         if (shutdown) {
1162                 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
1163                 kernel_power_off();
1164         }
1165         return firsterr;
1166 }
1167
1168 module_init(ref_scale_init);
1169 module_exit(ref_scale_cleanup);