lib/locking-selftest.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * lib/locking-selftest.c
   4  *
   5  * Testsuite for various locking APIs: spinlocks, rwlocks,
   6  * mutexes and rw-semaphores.
   7  *
   8  * It is checking both false positives and false negatives.
   9  *
  10  * Started by Ingo Molnar:
  11  *
  12  *  Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  13  */
  14 #include <linux/rwsem.h>
  15 #include <linux/mutex.h>
  16 #include <linux/ww_mutex.h>
  17 #include <linux/sched.h>
  18 #include <linux/sched/mm.h>
  19 #include <linux/delay.h>
  20 #include <linux/lockdep.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/kallsyms.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/debug_locks.h>
  25 #include <linux/irqflags.h>
  26 #include <linux/rtmutex.h>
  27 #include <linux/local_lock.h>
  28
  29 /*
  30  * Change this to 1 if you want to see the failure printouts:
  31  */
  32 static unsigned int debug_locks_verbose;
  33 unsigned int force_read_lock_recursive;
  34
  35 static DEFINE_WD_CLASS(ww_lockdep);
  36
  37 static int __init setup_debug_locks_verbose(char *str)
  38 {
  39         get_option(&str, &debug_locks_verbose);
  40
  41         return 1;
  42 }
  43
  44 __setup("debug_locks_verbose=", setup_debug_locks_verbose);
  45
  46 #define FAILURE         0
  47 #define SUCCESS         1
  48
  49 #define LOCKTYPE_SPIN   0x1
  50 #define LOCKTYPE_RWLOCK 0x2
  51 #define LOCKTYPE_MUTEX  0x4
  52 #define LOCKTYPE_RWSEM  0x8
  53 #define LOCKTYPE_WW     0x10
  54 #define LOCKTYPE_RTMUTEX 0x20
  55 #define LOCKTYPE_LL     0x40
  56 #define LOCKTYPE_SPECIAL 0x80
  57
  58 static struct ww_acquire_ctx t, t2;
  59 static struct ww_mutex o, o2, o3;
  60
  61 /*
  62  * Normal standalone locks, for the circular and irq-context
  63  * dependency tests:
  64  */
  65 static DEFINE_SPINLOCK(lock_A);
  66 static DEFINE_SPINLOCK(lock_B);
  67 static DEFINE_SPINLOCK(lock_C);
  68 static DEFINE_SPINLOCK(lock_D);
  69
  70 static DEFINE_RAW_SPINLOCK(raw_lock_A);
  71 static DEFINE_RAW_SPINLOCK(raw_lock_B);
  72
  73 static DEFINE_RWLOCK(rwlock_A);
  74 static DEFINE_RWLOCK(rwlock_B);
  75 static DEFINE_RWLOCK(rwlock_C);
  76 static DEFINE_RWLOCK(rwlock_D);
  77
  78 static DEFINE_MUTEX(mutex_A);
  79 static DEFINE_MUTEX(mutex_B);
  80 static DEFINE_MUTEX(mutex_C);
  81 static DEFINE_MUTEX(mutex_D);
  82
  83 static DECLARE_RWSEM(rwsem_A);
  84 static DECLARE_RWSEM(rwsem_B);
  85 static DECLARE_RWSEM(rwsem_C);
  86 static DECLARE_RWSEM(rwsem_D);
  87
  88 #ifdef CONFIG_RT_MUTEXES
  89
  90 static DEFINE_RT_MUTEX(rtmutex_A);
  91 static DEFINE_RT_MUTEX(rtmutex_B);
  92 static DEFINE_RT_MUTEX(rtmutex_C);
  93 static DEFINE_RT_MUTEX(rtmutex_D);
  94
  95 #endif
  96
  97 /*
  98  * Locks that we initialize dynamically as well so that
  99  * e.g. X1 and X2 becomes two instances of the same class,
 100  * but X* and Y* are different classes. We do this so that
 101  * we do not trigger a real lockup:
 102  */
 103 static DEFINE_SPINLOCK(lock_X1);
 104 static DEFINE_SPINLOCK(lock_X2);
 105 static DEFINE_SPINLOCK(lock_Y1);
 106 static DEFINE_SPINLOCK(lock_Y2);
 107 static DEFINE_SPINLOCK(lock_Z1);
 108 static DEFINE_SPINLOCK(lock_Z2);
 109
 110 static DEFINE_RWLOCK(rwlock_X1);
 111 static DEFINE_RWLOCK(rwlock_X2);
 112 static DEFINE_RWLOCK(rwlock_Y1);
 113 static DEFINE_RWLOCK(rwlock_Y2);
 114 static DEFINE_RWLOCK(rwlock_Z1);
 115 static DEFINE_RWLOCK(rwlock_Z2);
 116
 117 static DEFINE_MUTEX(mutex_X1);
 118 static DEFINE_MUTEX(mutex_X2);
 119 static DEFINE_MUTEX(mutex_Y1);
 120 static DEFINE_MUTEX(mutex_Y2);
 121 static DEFINE_MUTEX(mutex_Z1);
 122 static DEFINE_MUTEX(mutex_Z2);
 123
 124 static DECLARE_RWSEM(rwsem_X1);
 125 static DECLARE_RWSEM(rwsem_X2);
 126 static DECLARE_RWSEM(rwsem_Y1);
 127 static DECLARE_RWSEM(rwsem_Y2);
 128 static DECLARE_RWSEM(rwsem_Z1);
 129 static DECLARE_RWSEM(rwsem_Z2);
 130
 131 #ifdef CONFIG_RT_MUTEXES
 132
 133 static DEFINE_RT_MUTEX(rtmutex_X1);
 134 static DEFINE_RT_MUTEX(rtmutex_X2);
 135 static DEFINE_RT_MUTEX(rtmutex_Y1);
 136 static DEFINE_RT_MUTEX(rtmutex_Y2);
 137 static DEFINE_RT_MUTEX(rtmutex_Z1);
 138 static DEFINE_RT_MUTEX(rtmutex_Z2);
 139
 140 #endif
 141
 142 static local_lock_t local_A = INIT_LOCAL_LOCK(local_A);
 143
 144 /*
 145  * non-inlined runtime initializers, to let separate locks share
 146  * the same lock-class:
 147  */
 148 #define INIT_CLASS_FUNC(class)                          \
 149 static noinline void                                    \
 150 init_class_##class(spinlock_t *lock, rwlock_t *rwlock, \
 151         struct mutex *mutex, struct rw_semaphore *rwsem)\
 152 {                                                       \
 153         spin_lock_init(lock);                   \
 154         rwlock_init(rwlock);                            \
 155         mutex_init(mutex);                              \
 156         init_rwsem(rwsem);                              \
 157 }
 158
 159 INIT_CLASS_FUNC(X)
 160 INIT_CLASS_FUNC(Y)
 161 INIT_CLASS_FUNC(Z)
 162
 163 static void init_shared_classes(void)
 164 {
 165 #ifdef CONFIG_RT_MUTEXES
 166         static struct lock_class_key rt_X, rt_Y, rt_Z;
 167
 168         __rt_mutex_init(&rtmutex_X1, __func__, &rt_X);
 169         __rt_mutex_init(&rtmutex_X2, __func__, &rt_X);
 170         __rt_mutex_init(&rtmutex_Y1, __func__, &rt_Y);
 171         __rt_mutex_init(&rtmutex_Y2, __func__, &rt_Y);
 172         __rt_mutex_init(&rtmutex_Z1, __func__, &rt_Z);
 173         __rt_mutex_init(&rtmutex_Z2, __func__, &rt_Z);
 174 #endif
 175
 176         init_class_X(&lock_X1, &rwlock_X1, &mutex_X1, &rwsem_X1);
 177         init_class_X(&lock_X2, &rwlock_X2, &mutex_X2, &rwsem_X2);
 178
 179         init_class_Y(&lock_Y1, &rwlock_Y1, &mutex_Y1, &rwsem_Y1);
 180         init_class_Y(&lock_Y2, &rwlock_Y2, &mutex_Y2, &rwsem_Y2);
 181
 182         init_class_Z(&lock_Z1, &rwlock_Z1, &mutex_Z1, &rwsem_Z1);
 183         init_class_Z(&lock_Z2, &rwlock_Z2, &mutex_Z2, &rwsem_Z2);
 184 }
 185
 186 /*
 187  * For spinlocks and rwlocks we also do hardirq-safe / softirq-safe tests.
 188  * The following functions use a lock from a simulated hardirq/softirq
 189  * context, causing the locks to be marked as hardirq-safe/softirq-safe:
 190  */
 191
 192 #define HARDIRQ_DISABLE         local_irq_disable
 193 #define HARDIRQ_ENABLE          local_irq_enable
 194
 195 #define HARDIRQ_ENTER()                         \
 196         local_irq_disable();                    \
 197         __irq_enter();                          \
 198         lockdep_hardirq_threaded();             \
 199         WARN_ON(!in_irq());
 200
 201 #define HARDIRQ_EXIT()                          \
 202         __irq_exit();                           \
 203         local_irq_enable();
 204
 205 #define SOFTIRQ_DISABLE         local_bh_disable
 206 #define SOFTIRQ_ENABLE          local_bh_enable
 207
 208 #define SOFTIRQ_ENTER()                         \
 209                 local_bh_disable();             \
 210                 local_irq_disable();            \
 211                 lockdep_softirq_enter();        \
 212                 WARN_ON(!in_softirq());
 213
 214 #define SOFTIRQ_EXIT()                          \
 215                 lockdep_softirq_exit();         \
 216                 local_irq_enable();             \
 217                 local_bh_enable();
 218
 219 /*
 220  * Shortcuts for lock/unlock API variants, to keep
 221  * the testcases compact:
 222  */
 223 #define L(x)                    spin_lock(&lock_##x)
 224 #define U(x)                    spin_unlock(&lock_##x)
 225 #define LU(x)                   L(x); U(x)
 226 #define SI(x)                   spin_lock_init(&lock_##x)
 227
 228 #define WL(x)                   write_lock(&rwlock_##x)
 229 #define WU(x)                   write_unlock(&rwlock_##x)
 230 #define WLU(x)                  WL(x); WU(x)
 231
 232 #define RL(x)                   read_lock(&rwlock_##x)
 233 #define RU(x)                   read_unlock(&rwlock_##x)
 234 #define RLU(x)                  RL(x); RU(x)
 235 #define RWI(x)                  rwlock_init(&rwlock_##x)
 236
 237 #define ML(x)                   mutex_lock(&mutex_##x)
 238 #define MU(x)                   mutex_unlock(&mutex_##x)
 239 #define MI(x)                   mutex_init(&mutex_##x)
 240
 241 #define RTL(x)                  rt_mutex_lock(&rtmutex_##x)
 242 #define RTU(x)                  rt_mutex_unlock(&rtmutex_##x)
 243 #define RTI(x)                  rt_mutex_init(&rtmutex_##x)
 244
 245 #define WSL(x)                  down_write(&rwsem_##x)
 246 #define WSU(x)                  up_write(&rwsem_##x)
 247
 248 #define RSL(x)                  down_read(&rwsem_##x)
 249 #define RSU(x)                  up_read(&rwsem_##x)
 250 #define RWSI(x)                 init_rwsem(&rwsem_##x)
 251
 252 #ifndef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
 253 #define WWAI(x)                 ww_acquire_init(x, &ww_lockdep)
 254 #else
 255 #define WWAI(x)                 do { ww_acquire_init(x, &ww_lockdep); (x)->deadlock_inject_countdown = ~0U; } while (0)
 256 #endif
 257 #define WWAD(x)                 ww_acquire_done(x)
 258 #define WWAF(x)                 ww_acquire_fini(x)
 259
 260 #define WWL(x, c)               ww_mutex_lock(x, c)
 261 #define WWT(x)                  ww_mutex_trylock(x)
 262 #define WWL1(x)                 ww_mutex_lock(x, NULL)
 263 #define WWU(x)                  ww_mutex_unlock(x)
 264
 265
 266 #define LOCK_UNLOCK_2(x,y)      LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x)
 267
 268 /*
 269  * Generate different permutations of the same testcase, using
 270  * the same basic lock-dependency/state events:
 271  */
 272
 273 #define GENERATE_TESTCASE(name)                 \
 274                                                 \
 275 static void name(void) { E(); }
 276
 277 #define GENERATE_PERMUTATIONS_2_EVENTS(name)    \
 278                                                 \
 279 static void name##_12(void) { E1(); E2(); }     \
 280 static void name##_21(void) { E2(); E1(); }
 281
 282 #define GENERATE_PERMUTATIONS_3_EVENTS(name)            \
 283                                                         \
 284 static void name##_123(void) { E1(); E2(); E3(); }      \
 285 static void name##_132(void) { E1(); E3(); E2(); }      \
 286 static void name##_213(void) { E2(); E1(); E3(); }      \
 287 static void name##_231(void) { E2(); E3(); E1(); }      \
 288 static void name##_312(void) { E3(); E1(); E2(); }      \
 289 static void name##_321(void) { E3(); E2(); E1(); }
 290
 291 /*
 292  * AA deadlock:
 293  */
 294
 295 #define E()                                     \
 296                                                 \
 297         LOCK(X1);                               \
 298         LOCK(X2); /* this one should fail */
 299
 300 /*
 301  * 6 testcases:
 302  */
 303 #include "locking-selftest-spin.h"
 304 GENERATE_TESTCASE(AA_spin)
 305 #include "locking-selftest-wlock.h"
 306 GENERATE_TESTCASE(AA_wlock)
 307 #include "locking-selftest-rlock.h"
 308 GENERATE_TESTCASE(AA_rlock)
 309 #include "locking-selftest-mutex.h"
 310 GENERATE_TESTCASE(AA_mutex)
 311 #include "locking-selftest-wsem.h"
 312 GENERATE_TESTCASE(AA_wsem)
 313 #include "locking-selftest-rsem.h"
 314 GENERATE_TESTCASE(AA_rsem)
 315
 316 #ifdef CONFIG_RT_MUTEXES
 317 #include "locking-selftest-rtmutex.h"
 318 GENERATE_TESTCASE(AA_rtmutex);
 319 #endif
 320
 321 #undef E
 322
 323 /*
 324  * Special-case for read-locking, they are
 325  * allowed to recurse on the same lock class:
 326  */
 327 static void rlock_AA1(void)
 328 {
 329         RL(X1);
 330         RL(X1); // this one should NOT fail
 331 }
 332
 333 static void rlock_AA1B(void)
 334 {
 335         RL(X1);
 336         RL(X2); // this one should NOT fail
 337 }
 338
 339 static void rsem_AA1(void)
 340 {
 341         RSL(X1);
 342         RSL(X1); // this one should fail
 343 }
 344
 345 static void rsem_AA1B(void)
 346 {
 347         RSL(X1);
 348         RSL(X2); // this one should fail
 349 }
 350 /*
 351  * The mixing of read and write locks is not allowed:
 352  */
 353 static void rlock_AA2(void)
 354 {
 355         RL(X1);
 356         WL(X2); // this one should fail
 357 }
 358
 359 static void rsem_AA2(void)
 360 {
 361         RSL(X1);
 362         WSL(X2); // this one should fail
 363 }
 364
 365 static void rlock_AA3(void)
 366 {
 367         WL(X1);
 368         RL(X2); // this one should fail
 369 }
 370
 371 static void rsem_AA3(void)
 372 {
 373         WSL(X1);
 374         RSL(X2); // this one should fail
 375 }
 376
 377 /*
 378  * read_lock(A)
 379  * spin_lock(B)
 380  *              spin_lock(B)
 381  *              write_lock(A)
 382  */
 383 static void rlock_ABBA1(void)
 384 {
 385         RL(X1);
 386         L(Y1);
 387         U(Y1);
 388         RU(X1);
 389
 390         L(Y1);
 391         WL(X1);
 392         WU(X1);
 393         U(Y1); // should fail
 394 }
 395
 396 static void rwsem_ABBA1(void)
 397 {
 398         RSL(X1);
 399         ML(Y1);
 400         MU(Y1);
 401         RSU(X1);
 402
 403         ML(Y1);
 404         WSL(X1);
 405         WSU(X1);
 406         MU(Y1); // should fail
 407 }
 408
 409 /*
 410  * read_lock(A)
 411  * spin_lock(B)
 412  *              spin_lock(B)
 413  *              write_lock(A)
 414  *
 415  * This test case is aimed at poking whether the chain cache prevents us from
 416  * detecting a read-lock/lock-write deadlock: if the chain cache doesn't differ
 417  * read/write locks, the following case may happen
 418  *
 419  *      { read_lock(A)->lock(B) dependency exists }
 420  *
 421  *      P0:
 422  *      lock(B);
 423  *      read_lock(A);
 424  *
 425  *      { Not a deadlock, B -> A is added in the chain cache }
 426  *
 427  *      P1:
 428  *      lock(B);
 429  *      write_lock(A);
 430  *
 431  *      { B->A found in chain cache, not reported as a deadlock }
 432  *
 433  */
 434 static void rlock_chaincache_ABBA1(void)
 435 {
 436         RL(X1);
 437         L(Y1);
 438         U(Y1);
 439         RU(X1);
 440
 441         L(Y1);
 442         RL(X1);
 443         RU(X1);
 444         U(Y1);
 445
 446         L(Y1);
 447         WL(X1);
 448         WU(X1);
 449         U(Y1); // should fail
 450 }
 451
 452 /*
 453  * read_lock(A)
 454  * spin_lock(B)
 455  *              spin_lock(B)
 456  *              read_lock(A)
 457  */
 458 static void rlock_ABBA2(void)
 459 {
 460         RL(X1);
 461         L(Y1);
 462         U(Y1);
 463         RU(X1);
 464
 465         L(Y1);
 466         RL(X1);
 467         RU(X1);
 468         U(Y1); // should NOT fail
 469 }
 470
 471 static void rwsem_ABBA2(void)
 472 {
 473         RSL(X1);
 474         ML(Y1);
 475         MU(Y1);
 476         RSU(X1);
 477
 478         ML(Y1);
 479         RSL(X1);
 480         RSU(X1);
 481         MU(Y1); // should fail
 482 }
 483
 484
 485 /*
 486  * write_lock(A)
 487  * spin_lock(B)
 488  *              spin_lock(B)
 489  *              write_lock(A)
 490  */
 491 static void rlock_ABBA3(void)
 492 {
 493         WL(X1);
 494         L(Y1);
 495         U(Y1);
 496         WU(X1);
 497
 498         L(Y1);
 499         WL(X1);
 500         WU(X1);
 501         U(Y1); // should fail
 502 }
 503
 504 static void rwsem_ABBA3(void)
 505 {
 506         WSL(X1);
 507         ML(Y1);
 508         MU(Y1);
 509         WSU(X1);
 510
 511         ML(Y1);
 512         WSL(X1);
 513         WSU(X1);
 514         MU(Y1); // should fail
 515 }
 516
 517 /*
 518  * ABBA deadlock:
 519  */
 520
 521 #define E()                                     \
 522                                                 \
 523         LOCK_UNLOCK_2(A, B);                    \
 524         LOCK_UNLOCK_2(B, A); /* fail */
 525
 526 /*
 527  * 6 testcases:
 528  */
 529 #include "locking-selftest-spin.h"
 530 GENERATE_TESTCASE(ABBA_spin)
 531 #include "locking-selftest-wlock.h"
 532 GENERATE_TESTCASE(ABBA_wlock)
 533 #include "locking-selftest-rlock.h"
 534 GENERATE_TESTCASE(ABBA_rlock)
 535 #include "locking-selftest-mutex.h"
 536 GENERATE_TESTCASE(ABBA_mutex)
 537 #include "locking-selftest-wsem.h"
 538 GENERATE_TESTCASE(ABBA_wsem)
 539 #include "locking-selftest-rsem.h"
 540 GENERATE_TESTCASE(ABBA_rsem)
 541
 542 #ifdef CONFIG_RT_MUTEXES
 543 #include "locking-selftest-rtmutex.h"
 544 GENERATE_TESTCASE(ABBA_rtmutex);
 545 #endif
 546
 547 #undef E
 548
 549 /*
 550  * AB BC CA deadlock:
 551  */
 552
 553 #define E()                                     \
 554                                                 \
 555         LOCK_UNLOCK_2(A, B);                    \
 556         LOCK_UNLOCK_2(B, C);                    \
 557         LOCK_UNLOCK_2(C, A); /* fail */
 558
 559 /*
 560  * 6 testcases:
 561  */
 562 #include "locking-selftest-spin.h"
 563 GENERATE_TESTCASE(ABBCCA_spin)
 564 #include "locking-selftest-wlock.h"
 565 GENERATE_TESTCASE(ABBCCA_wlock)
 566 #include "locking-selftest-rlock.h"
 567 GENERATE_TESTCASE(ABBCCA_rlock)
 568 #include "locking-selftest-mutex.h"
 569 GENERATE_TESTCASE(ABBCCA_mutex)
 570 #include "locking-selftest-wsem.h"
 571 GENERATE_TESTCASE(ABBCCA_wsem)
 572 #include "locking-selftest-rsem.h"
 573 GENERATE_TESTCASE(ABBCCA_rsem)
 574
 575 #ifdef CONFIG_RT_MUTEXES
 576 #include "locking-selftest-rtmutex.h"
 577 GENERATE_TESTCASE(ABBCCA_rtmutex);
 578 #endif
 579
 580 #undef E
 581
 582 /*
 583  * AB CA BC deadlock:
 584  */
 585
 586 #define E()                                     \
 587                                                 \
 588         LOCK_UNLOCK_2(A, B);                    \
 589         LOCK_UNLOCK_2(C, A);                    \
 590         LOCK_UNLOCK_2(B, C); /* fail */
 591
 592 /*
 593  * 6 testcases:
 594  */
 595 #include "locking-selftest-spin.h"
 596 GENERATE_TESTCASE(ABCABC_spin)
 597 #include "locking-selftest-wlock.h"
 598 GENERATE_TESTCASE(ABCABC_wlock)
 599 #include "locking-selftest-rlock.h"
 600 GENERATE_TESTCASE(ABCABC_rlock)
 601 #include "locking-selftest-mutex.h"
 602 GENERATE_TESTCASE(ABCABC_mutex)
 603 #include "locking-selftest-wsem.h"
 604 GENERATE_TESTCASE(ABCABC_wsem)
 605 #include "locking-selftest-rsem.h"
 606 GENERATE_TESTCASE(ABCABC_rsem)
 607
 608 #ifdef CONFIG_RT_MUTEXES
 609 #include "locking-selftest-rtmutex.h"
 610 GENERATE_TESTCASE(ABCABC_rtmutex);
 611 #endif
 612
 613 #undef E
 614
 615 /*
 616  * AB BC CD DA deadlock:
 617  */
 618
 619 #define E()                                     \
 620                                                 \
 621         LOCK_UNLOCK_2(A, B);                    \
 622         LOCK_UNLOCK_2(B, C);                    \
 623         LOCK_UNLOCK_2(C, D);                    \
 624         LOCK_UNLOCK_2(D, A); /* fail */
 625
 626 /*
 627  * 6 testcases:
 628  */
 629 #include "locking-selftest-spin.h"
 630 GENERATE_TESTCASE(ABBCCDDA_spin)
 631 #include "locking-selftest-wlock.h"
 632 GENERATE_TESTCASE(ABBCCDDA_wlock)
 633 #include "locking-selftest-rlock.h"
 634 GENERATE_TESTCASE(ABBCCDDA_rlock)
 635 #include "locking-selftest-mutex.h"
 636 GENERATE_TESTCASE(ABBCCDDA_mutex)
 637 #include "locking-selftest-wsem.h"
 638 GENERATE_TESTCASE(ABBCCDDA_wsem)
 639 #include "locking-selftest-rsem.h"
 640 GENERATE_TESTCASE(ABBCCDDA_rsem)
 641
 642 #ifdef CONFIG_RT_MUTEXES
 643 #include "locking-selftest-rtmutex.h"
 644 GENERATE_TESTCASE(ABBCCDDA_rtmutex);
 645 #endif
 646
 647 #undef E
 648
 649 /*
 650  * AB CD BD DA deadlock:
 651  */
 652 #define E()                                     \
 653                                                 \
 654         LOCK_UNLOCK_2(A, B);                    \
 655         LOCK_UNLOCK_2(C, D);                    \
 656         LOCK_UNLOCK_2(B, D);                    \
 657         LOCK_UNLOCK_2(D, A); /* fail */
 658
 659 /*
 660  * 6 testcases:
 661  */
 662 #include "locking-selftest-spin.h"
 663 GENERATE_TESTCASE(ABCDBDDA_spin)
 664 #include "locking-selftest-wlock.h"
 665 GENERATE_TESTCASE(ABCDBDDA_wlock)
 666 #include "locking-selftest-rlock.h"
 667 GENERATE_TESTCASE(ABCDBDDA_rlock)
 668 #include "locking-selftest-mutex.h"
 669 GENERATE_TESTCASE(ABCDBDDA_mutex)
 670 #include "locking-selftest-wsem.h"
 671 GENERATE_TESTCASE(ABCDBDDA_wsem)
 672 #include "locking-selftest-rsem.h"
 673 GENERATE_TESTCASE(ABCDBDDA_rsem)
 674
 675 #ifdef CONFIG_RT_MUTEXES
 676 #include "locking-selftest-rtmutex.h"
 677 GENERATE_TESTCASE(ABCDBDDA_rtmutex);
 678 #endif
 679
 680 #undef E
 681
 682 /*
 683  * AB CD BC DA deadlock:
 684  */
 685 #define E()                                     \
 686                                                 \
 687         LOCK_UNLOCK_2(A, B);                    \
 688         LOCK_UNLOCK_2(C, D);                    \
 689         LOCK_UNLOCK_2(B, C);                    \
 690         LOCK_UNLOCK_2(D, A); /* fail */
 691
 692 /*
 693  * 6 testcases:
 694  */
 695 #include "locking-selftest-spin.h"
 696 GENERATE_TESTCASE(ABCDBCDA_spin)
 697 #include "locking-selftest-wlock.h"
 698 GENERATE_TESTCASE(ABCDBCDA_wlock)
 699 #include "locking-selftest-rlock.h"
 700 GENERATE_TESTCASE(ABCDBCDA_rlock)
 701 #include "locking-selftest-mutex.h"
 702 GENERATE_TESTCASE(ABCDBCDA_mutex)
 703 #include "locking-selftest-wsem.h"
 704 GENERATE_TESTCASE(ABCDBCDA_wsem)
 705 #include "locking-selftest-rsem.h"
 706 GENERATE_TESTCASE(ABCDBCDA_rsem)
 707
 708 #ifdef CONFIG_RT_MUTEXES
 709 #include "locking-selftest-rtmutex.h"
 710 GENERATE_TESTCASE(ABCDBCDA_rtmutex);
 711 #endif
 712
 713 #undef E
 714
 715 /*
 716  * Double unlock:
 717  */
 718 #define E()                                     \
 719                                                 \
 720         LOCK(A);                                \
 721         UNLOCK(A);                              \
 722         UNLOCK(A); /* fail */
 723
 724 /*
 725  * 6 testcases:
 726  */
 727 #include "locking-selftest-spin.h"
 728 GENERATE_TESTCASE(double_unlock_spin)
 729 #include "locking-selftest-wlock.h"
 730 GENERATE_TESTCASE(double_unlock_wlock)
 731 #include "locking-selftest-rlock.h"
 732 GENERATE_TESTCASE(double_unlock_rlock)
 733 #include "locking-selftest-mutex.h"
 734 GENERATE_TESTCASE(double_unlock_mutex)
 735 #include "locking-selftest-wsem.h"
 736 GENERATE_TESTCASE(double_unlock_wsem)
 737 #include "locking-selftest-rsem.h"
 738 GENERATE_TESTCASE(double_unlock_rsem)
 739
 740 #ifdef CONFIG_RT_MUTEXES
 741 #include "locking-selftest-rtmutex.h"
 742 GENERATE_TESTCASE(double_unlock_rtmutex);
 743 #endif
 744
 745 #undef E
 746
 747 /*
 748  * initializing a held lock:
 749  */
 750 #define E()                                     \
 751                                                 \
 752         LOCK(A);                                \
 753         INIT(A); /* fail */
 754
 755 /*
 756  * 6 testcases:
 757  */
 758 #include "locking-selftest-spin.h"
 759 GENERATE_TESTCASE(init_held_spin)
 760 #include "locking-selftest-wlock.h"
 761 GENERATE_TESTCASE(init_held_wlock)
 762 #include "locking-selftest-rlock.h"
 763 GENERATE_TESTCASE(init_held_rlock)
 764 #include "locking-selftest-mutex.h"
 765 GENERATE_TESTCASE(init_held_mutex)
 766 #include "locking-selftest-wsem.h"
 767 GENERATE_TESTCASE(init_held_wsem)
 768 #include "locking-selftest-rsem.h"
 769 GENERATE_TESTCASE(init_held_rsem)
 770
 771 #ifdef CONFIG_RT_MUTEXES
 772 #include "locking-selftest-rtmutex.h"
 773 GENERATE_TESTCASE(init_held_rtmutex);
 774 #endif
 775
 776 #undef E
 777
 778 /*
 779  * locking an irq-safe lock with irqs enabled:
 780  */
 781 #define E1()                            \
 782                                         \
 783         IRQ_ENTER();                    \
 784         LOCK(A);                        \
 785         UNLOCK(A);                      \
 786         IRQ_EXIT();
 787
 788 #define E2()                            \
 789                                         \
 790         LOCK(A);                        \
 791         UNLOCK(A);
 792
 793 /*
 794  * Generate 24 testcases:
 795  */
 796 #include "locking-selftest-spin-hardirq.h"
 797 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
 798
 799 #include "locking-selftest-rlock-hardirq.h"
 800 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
 801
 802 #include "locking-selftest-wlock-hardirq.h"
 803 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_wlock)
 804
 805 #include "locking-selftest-spin-softirq.h"
 806 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_spin)
 807
 808 #include "locking-selftest-rlock-softirq.h"
 809 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_rlock)
 810
 811 #include "locking-selftest-wlock-softirq.h"
 812 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
 813
 814 #undef E1
 815 #undef E2
 816
 817 /*
 818  * Enabling hardirqs with a softirq-safe lock held:
 819  */
 820 #define E1()                            \
 821                                         \
 822         SOFTIRQ_ENTER();                \
 823         LOCK(A);                        \
 824         UNLOCK(A);                      \
 825         SOFTIRQ_EXIT();
 826
 827 #define E2()                            \
 828                                         \
 829         HARDIRQ_DISABLE();              \
 830         LOCK(A);                        \
 831         HARDIRQ_ENABLE();               \
 832         UNLOCK(A);
 833
 834 /*
 835  * Generate 12 testcases:
 836  */
 837 #include "locking-selftest-spin.h"
 838 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_spin)
 839
 840 #include "locking-selftest-wlock.h"
 841 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_wlock)
 842
 843 #include "locking-selftest-rlock.h"
 844 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock)
 845
 846 #undef E1
 847 #undef E2
 848
 849 /*
 850  * Enabling irqs with an irq-safe lock held:
 851  */
 852 #define E1()                            \
 853                                         \
 854         IRQ_ENTER();                    \
 855         LOCK(A);                        \
 856         UNLOCK(A);                      \
 857         IRQ_EXIT();
 858
 859 #define E2()                            \
 860                                         \
 861         IRQ_DISABLE();                  \
 862         LOCK(A);                        \
 863         IRQ_ENABLE();                   \
 864         UNLOCK(A);
 865
 866 /*
 867  * Generate 24 testcases:
 868  */
 869 #include "locking-selftest-spin-hardirq.h"
 870 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
 871
 872 #include "locking-selftest-rlock-hardirq.h"
 873 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
 874
 875 #include "locking-selftest-wlock-hardirq.h"
 876 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_wlock)
 877
 878 #include "locking-selftest-spin-softirq.h"
 879 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_spin)
 880
 881 #include "locking-selftest-rlock-softirq.h"
 882 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_rlock)
 883
 884 #include "locking-selftest-wlock-softirq.h"
 885 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
 886
 887 #undef E1
 888 #undef E2
 889
 890 /*
 891  * Acquiring a irq-unsafe lock while holding an irq-safe-lock:
 892  */
 893 #define E1()                            \
 894                                         \
 895         LOCK(A);                        \
 896         LOCK(B);                        \
 897         UNLOCK(B);                      \
 898         UNLOCK(A);                      \
 899
 900 #define E2()                            \
 901                                         \
 902         LOCK(B);                        \
 903         UNLOCK(B);
 904
 905 #define E3()                            \
 906                                         \
 907         IRQ_ENTER();                    \
 908         LOCK(A);                        \
 909         UNLOCK(A);                      \
 910         IRQ_EXIT();
 911
 912 /*
 913  * Generate 36 testcases:
 914  */
 915 #include "locking-selftest-spin-hardirq.h"
 916 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
 917
 918 #include "locking-selftest-rlock-hardirq.h"
 919 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
 920
 921 #include "locking-selftest-wlock-hardirq.h"
 922 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_wlock)
 923
 924 #include "locking-selftest-spin-softirq.h"
 925 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_spin)
 926
 927 #include "locking-selftest-rlock-softirq.h"
 928 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_rlock)
 929
 930 #include "locking-selftest-wlock-softirq.h"
 931 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
 932
 933 #undef E1
 934 #undef E2
 935 #undef E3
 936
 937 /*
 938  * If a lock turns into softirq-safe, but earlier it took
 939  * a softirq-unsafe lock:
 940  */
 941
 942 #define E1()                            \
 943         IRQ_DISABLE();                  \
 944         LOCK(A);                        \
 945         LOCK(B);                        \
 946         UNLOCK(B);                      \
 947         UNLOCK(A);                      \
 948         IRQ_ENABLE();
 949
 950 #define E2()                            \
 951         LOCK(B);                        \
 952         UNLOCK(B);
 953
 954 #define E3()                            \
 955         IRQ_ENTER();                    \
 956         LOCK(A);                        \
 957         UNLOCK(A);                      \
 958         IRQ_EXIT();
 959
 960 /*
 961  * Generate 36 testcases:
 962  */
 963 #include "locking-selftest-spin-hardirq.h"
 964 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
 965
 966 #include "locking-selftest-rlock-hardirq.h"
 967 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
 968
 969 #include "locking-selftest-wlock-hardirq.h"
 970 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_wlock)
 971
 972 #include "locking-selftest-spin-softirq.h"
 973 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_spin)
 974
 975 #include "locking-selftest-rlock-softirq.h"
 976 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_rlock)
 977
 978 #include "locking-selftest-wlock-softirq.h"
 979 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
 980
 981 #undef E1
 982 #undef E2
 983 #undef E3
 984
 985 /*
 986  * read-lock / write-lock irq inversion.
 987  *
 988  * Deadlock scenario:
 989  *
 990  * CPU#1 is at #1, i.e. it has write-locked A, but has not
 991  * taken B yet.
 992  *
 993  * CPU#2 is at #2, i.e. it has locked B.
 994  *
 995  * Hardirq hits CPU#2 at point #2 and is trying to read-lock A.
 996  *
 997  * The deadlock occurs because CPU#1 will spin on B, and CPU#2
 998  * will spin on A.
 999  */
1000
1001 #define E1()                            \
1002                                         \
1003         IRQ_DISABLE();                  \
1004         WL(A);                          \
1005         LOCK(B);                        \
1006         UNLOCK(B);                      \
1007         WU(A);                          \
1008         IRQ_ENABLE();
1009
1010 #define E2()                            \
1011                                         \
1012         LOCK(B);                        \
1013         UNLOCK(B);
1014
1015 #define E3()                            \
1016                                         \
1017         IRQ_ENTER();                    \
1018         RL(A);                          \
1019         RU(A);                          \
1020         IRQ_EXIT();
1021
1022 /*
1023  * Generate 36 testcases:
1024  */
1025 #include "locking-selftest-spin-hardirq.h"
1026 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_spin)
1027
1028 #include "locking-selftest-rlock-hardirq.h"
1029 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_rlock)
1030
1031 #include "locking-selftest-wlock-hardirq.h"
1032 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_wlock)
1033
1034 #include "locking-selftest-spin-softirq.h"
1035 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_spin)
1036
1037 #include "locking-selftest-rlock-softirq.h"
1038 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_rlock)
1039
1040 #include "locking-selftest-wlock-softirq.h"
1041 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock)
1042
1043 #undef E1
1044 #undef E2
1045 #undef E3
1046
1047 /*
1048  * write-read / write-read / write-read deadlock even if read is recursive
1049  */
1050
1051 #define E1()                            \
1052                                         \
1053         WL(X1);                         \
1054         RL(Y1);                         \
1055         RU(Y1);                         \
1056         WU(X1);
1057
1058 #define E2()                            \
1059                                         \
1060         WL(Y1);                         \
1061         RL(Z1);                         \
1062         RU(Z1);                         \
1063         WU(Y1);
1064
1065 #define E3()                            \
1066                                         \
1067         WL(Z1);                         \
1068         RL(X1);                         \
1069         RU(X1);                         \
1070         WU(Z1);
1071
1072 #include "locking-selftest-rlock.h"
1073 GENERATE_PERMUTATIONS_3_EVENTS(W1R2_W2R3_W3R1)
1074
1075 #undef E1
1076 #undef E2
1077 #undef E3
1078
1079 /*
1080  * write-write / read-read / write-read deadlock even if read is recursive
1081  */
1082
1083 #define E1()                            \
1084                                         \
1085         WL(X1);                         \
1086         WL(Y1);                         \
1087         WU(Y1);                         \
1088         WU(X1);
1089
1090 #define E2()                            \
1091                                         \
1092         RL(Y1);                         \
1093         RL(Z1);                         \
1094         RU(Z1);                         \
1095         RU(Y1);
1096
1097 #define E3()                            \
1098                                         \
1099         WL(Z1);                         \
1100         RL(X1);                         \
1101         RU(X1);                         \
1102         WU(Z1);
1103
1104 #include "locking-selftest-rlock.h"
1105 GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_W3R1)
1106
1107 #undef E1
1108 #undef E2
1109 #undef E3
1110
1111 /*
1112  * write-write / read-read / read-write is not deadlock when read is recursive
1113  */
1114
1115 #define E1()                            \
1116                                         \
1117         WL(X1);                         \
1118         WL(Y1);                         \
1119         WU(Y1);                         \
1120         WU(X1);
1121
1122 #define E2()                            \
1123                                         \
1124         RL(Y1);                         \
1125         RL(Z1);                         \
1126         RU(Z1);                         \
1127         RU(Y1);
1128
1129 #define E3()                            \
1130                                         \
1131         RL(Z1);                         \
1132         WL(X1);                         \
1133         WU(X1);                         \
1134         RU(Z1);
1135
1136 #include "locking-selftest-rlock.h"
1137 GENERATE_PERMUTATIONS_3_EVENTS(W1R2_R2R3_W3W1)
1138
1139 #undef E1
1140 #undef E2
1141 #undef E3
1142
1143 /*
1144  * write-read / read-read / write-write is not deadlock when read is recursive
1145  */
1146
1147 #define E1()                            \
1148                                         \
1149         WL(X1);                         \
1150         RL(Y1);                         \
1151         RU(Y1);                         \
1152         WU(X1);
1153
1154 #define E2()                            \
1155                                         \
1156         RL(Y1);                         \
1157         RL(Z1);                         \
1158         RU(Z1);                         \
1159         RU(Y1);
1160
1161 #define E3()                            \
1162                                         \
1163         WL(Z1);                         \
1164         WL(X1);                         \
1165         WU(X1);                         \
1166         WU(Z1);
1167
1168 #include "locking-selftest-rlock.h"
1169 GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_R3W1)
1170
1171 #undef E1
1172 #undef E2
1173 #undef E3
1174 /*
1175  * read-lock / write-lock recursion that is actually safe.
1176  */
1177
1178 #define E1()                            \
1179                                         \
1180         IRQ_DISABLE();                  \
1181         WL(A);                          \
1182         WU(A);                          \
1183         IRQ_ENABLE();
1184
1185 #define E2()                            \
1186                                         \
1187         RL(A);                          \
1188         RU(A);                          \
1189
1190 #define E3()                            \
1191                                         \
1192         IRQ_ENTER();                    \
1193         LOCK(A);                        \
1194         L(B);                           \
1195         U(B);                           \
1196         UNLOCK(A);                      \
1197         IRQ_EXIT();
1198
1199 /*
1200  * Generate 24 testcases:
1201  */
1202 #include "locking-selftest-hardirq.h"
1203 #include "locking-selftest-rlock.h"
1204 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_rlock)
1205
1206 #include "locking-selftest-wlock.h"
1207 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_wlock)
1208
1209 #include "locking-selftest-softirq.h"
1210 #include "locking-selftest-rlock.h"
1211 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_rlock)
1212
1213 #include "locking-selftest-wlock.h"
1214 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_wlock)
1215
1216 #undef E1
1217 #undef E2
1218 #undef E3
1219
1220 /*
1221  * read-lock / write-lock recursion that is unsafe.
1222  */
1223
1224 #define E1()                            \
1225                                         \
1226         IRQ_DISABLE();                  \
1227         L(B);                           \
1228         LOCK(A);                        \
1229         UNLOCK(A);                      \
1230         U(B);                           \
1231         IRQ_ENABLE();
1232
1233 #define E2()                            \
1234                                         \
1235         RL(A);                          \
1236         RU(A);                          \
1237
1238 #define E3()                            \
1239                                         \
1240         IRQ_ENTER();                    \
1241         L(B);                           \
1242         U(B);                           \
1243         IRQ_EXIT();
1244
1245 /*
1246  * Generate 24 testcases:
1247  */
1248 #include "locking-selftest-hardirq.h"
1249 #include "locking-selftest-rlock.h"
1250 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_rlock)
1251
1252 #include "locking-selftest-wlock.h"
1253 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_wlock)
1254
1255 #include "locking-selftest-softirq.h"
1256 #include "locking-selftest-rlock.h"
1257 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_rlock)
1258
1259 #include "locking-selftest-wlock.h"
1260 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_wlock)
1261
1262 #undef E1
1263 #undef E2
1264 #undef E3
1265 /*
1266  * read-lock / write-lock recursion that is unsafe.
1267  *
1268  * A is a ENABLED_*_READ lock
1269  * B is a USED_IN_*_READ lock
1270  *
1271  * read_lock(A);
1272  *                      write_lock(B);
1273  * <interrupt>
1274  * read_lock(B);
1275  *                      write_lock(A); // if this one is read_lock(), no deadlock
1276  */
1277
1278 #define E1()                            \
1279                                         \
1280         IRQ_DISABLE();                  \
1281         WL(B);                          \
1282         LOCK(A);                        \
1283         UNLOCK(A);                      \
1284         WU(B);                          \
1285         IRQ_ENABLE();
1286
1287 #define E2()                            \
1288                                         \
1289         RL(A);                          \
1290         RU(A);                          \
1291
1292 #define E3()                            \
1293                                         \
1294         IRQ_ENTER();                    \
1295         RL(B);                          \
1296         RU(B);                          \
1297         IRQ_EXIT();
1298
1299 /*
1300  * Generate 24 testcases:
1301  */
1302 #include "locking-selftest-hardirq.h"
1303 #include "locking-selftest-rlock.h"
1304 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_rlock)
1305
1306 #include "locking-selftest-wlock.h"
1307 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_wlock)
1308
1309 #include "locking-selftest-softirq.h"
1310 #include "locking-selftest-rlock.h"
1311 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_rlock)
1312
1313 #include "locking-selftest-wlock.h"
1314 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_wlock)
1315
1316 #ifdef CONFIG_DEBUG_LOCK_ALLOC
1317 # define I_SPINLOCK(x)  lockdep_reset_lock(&lock_##x.dep_map)
1318 # define I_RAW_SPINLOCK(x)      lockdep_reset_lock(&raw_lock_##x.dep_map)
1319 # define I_RWLOCK(x)    lockdep_reset_lock(&rwlock_##x.dep_map)
1320 # define I_MUTEX(x)     lockdep_reset_lock(&mutex_##x.dep_map)
1321 # define I_RWSEM(x)     lockdep_reset_lock(&rwsem_##x.dep_map)
1322 # define I_WW(x)        lockdep_reset_lock(&x.dep_map)
1323 # define I_LOCAL_LOCK(x) lockdep_reset_lock(&local_##x.dep_map)
1324 #ifdef CONFIG_RT_MUTEXES
1325 # define I_RTMUTEX(x)   lockdep_reset_lock(&rtmutex_##x.dep_map)
1326 #endif
1327 #else
1328 # define I_SPINLOCK(x)
1329 # define I_RAW_SPINLOCK(x)
1330 # define I_RWLOCK(x)
1331 # define I_MUTEX(x)
1332 # define I_RWSEM(x)
1333 # define I_WW(x)
1334 # define I_LOCAL_LOCK(x)
1335 #endif
1336
1337 #ifndef I_RTMUTEX
1338 # define I_RTMUTEX(x)
1339 #endif
1340
1341 #ifdef CONFIG_RT_MUTEXES
1342 #define I2_RTMUTEX(x)   rt_mutex_init(&rtmutex_##x)
1343 #else
1344 #define I2_RTMUTEX(x)
1345 #endif
1346
1347 #define I1(x)                                   \
1348         do {                                    \
1349                 I_SPINLOCK(x);                  \
1350                 I_RWLOCK(x);                    \
1351                 I_MUTEX(x);                     \
1352                 I_RWSEM(x);                     \
1353                 I_RTMUTEX(x);                   \
1354         } while (0)
1355
1356 #define I2(x)                                   \
1357         do {                                    \
1358                 spin_lock_init(&lock_##x);      \
1359                 rwlock_init(&rwlock_##x);       \
1360                 mutex_init(&mutex_##x);         \
1361                 init_rwsem(&rwsem_##x);         \
1362                 I2_RTMUTEX(x);                  \
1363         } while (0)
1364
1365 static void reset_locks(void)
1366 {
1367         local_irq_disable();
1368         lockdep_free_key_range(&ww_lockdep.acquire_key, 1);
1369         lockdep_free_key_range(&ww_lockdep.mutex_key, 1);
1370
1371         I1(A); I1(B); I1(C); I1(D);
1372         I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2);
1373         I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base);
1374         I_RAW_SPINLOCK(A); I_RAW_SPINLOCK(B);
1375         I_LOCAL_LOCK(A);
1376
1377         lockdep_reset();
1378
1379         I2(A); I2(B); I2(C); I2(D);
1380         init_shared_classes();
1381         raw_spin_lock_init(&raw_lock_A);
1382         raw_spin_lock_init(&raw_lock_B);
1383         local_lock_init(&local_A);
1384
1385         ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep);
1386         memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2));
1387         memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key));
1388         memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key));
1389         local_irq_enable();
1390 }
1391
1392 #undef I
1393
1394 static int testcase_total;
1395 static int testcase_successes;
1396 static int expected_testcase_failures;
1397 static int unexpected_testcase_failures;
1398
1399 static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
1400 {
1401         unsigned long saved_preempt_count = preempt_count();
1402
1403         WARN_ON(irqs_disabled());
1404
1405         debug_locks_silent = !(debug_locks_verbose & lockclass_mask);
1406
1407         testcase_fn();
1408         /*
1409          * Filter out expected failures:
1410          */
1411 #ifndef CONFIG_PROVE_LOCKING
1412         if (expected == FAILURE && debug_locks) {
1413                 expected_testcase_failures++;
1414                 pr_cont("failed|");
1415         }
1416         else
1417 #endif
1418         if (debug_locks != expected) {
1419                 unexpected_testcase_failures++;
1420                 pr_cont("FAILED|");
1421         } else {
1422                 testcase_successes++;
1423                 pr_cont("  ok  |");
1424         }
1425         testcase_total++;
1426
1427         if (debug_locks_verbose & lockclass_mask)
1428                 pr_cont(" lockclass mask: %x, debug_locks: %d, expected: %d\n",
1429                         lockclass_mask, debug_locks, expected);
1430         /*
1431          * Some tests (e.g. double-unlock) might corrupt the preemption
1432          * count, so restore it:
1433          */
1434         preempt_count_set(saved_preempt_count);
1435 #ifdef CONFIG_TRACE_IRQFLAGS
1436         if (softirq_count())
1437                 current->softirqs_enabled = 0;
1438         else
1439                 current->softirqs_enabled = 1;
1440 #endif
1441
1442         reset_locks();
1443 }
1444
1445 #ifdef CONFIG_RT_MUTEXES
1446 #define dotest_rt(fn, e, m)     dotest((fn), (e), (m))
1447 #else
1448 #define dotest_rt(fn, e, m)
1449 #endif
1450
1451 static inline void print_testname(const char *testname)
1452 {
1453         printk("%33s:", testname);
1454 }
1455
1456 #define DO_TESTCASE_1(desc, name, nr)                           \
1457         print_testname(desc"/"#nr);                             \
1458         dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK);          \
1459         pr_cont("\n");
1460
1461 #define DO_TESTCASE_1B(desc, name, nr)                          \
1462         print_testname(desc"/"#nr);                             \
1463         dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK);          \
1464         pr_cont("\n");
1465
1466 #define DO_TESTCASE_1RR(desc, name, nr)                         \
1467         print_testname(desc"/"#nr);                             \
1468         pr_cont("             |");                              \
1469         dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK);          \
1470         pr_cont("\n");
1471
1472 #define DO_TESTCASE_1RRB(desc, name, nr)                        \
1473         print_testname(desc"/"#nr);                             \
1474         pr_cont("             |");                              \
1475         dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK);          \
1476         pr_cont("\n");
1477
1478
1479 #define DO_TESTCASE_3(desc, name, nr)                           \
1480         print_testname(desc"/"#nr);                             \
1481         dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN);       \
1482         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1483         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1484         pr_cont("\n");
1485
1486 #define DO_TESTCASE_3RW(desc, name, nr)                         \
1487         print_testname(desc"/"#nr);                             \
1488         dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN|LOCKTYPE_RWLOCK);\
1489         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1490         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1491         pr_cont("\n");
1492
1493 #define DO_TESTCASE_2RW(desc, name, nr)                         \
1494         print_testname(desc"/"#nr);                             \
1495         pr_cont("      |");                                     \
1496         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1497         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1498         pr_cont("\n");
1499
1500 #define DO_TESTCASE_2x2RW(desc, name, nr)                       \
1501         DO_TESTCASE_2RW("hard-"desc, name##_hard, nr)           \
1502         DO_TESTCASE_2RW("soft-"desc, name##_soft, nr)           \
1503
1504 #define DO_TESTCASE_6x2x2RW(desc, name)                         \
1505         DO_TESTCASE_2x2RW(desc, name, 123);                     \
1506         DO_TESTCASE_2x2RW(desc, name, 132);                     \
1507         DO_TESTCASE_2x2RW(desc, name, 213);                     \
1508         DO_TESTCASE_2x2RW(desc, name, 231);                     \
1509         DO_TESTCASE_2x2RW(desc, name, 312);                     \
1510         DO_TESTCASE_2x2RW(desc, name, 321);
1511
1512 #define DO_TESTCASE_6(desc, name)                               \
1513         print_testname(desc);                                   \
1514         dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);            \
1515         dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);         \
1516         dotest(name##_rlock, FAILURE, LOCKTYPE_RWLOCK);         \
1517         dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);          \
1518         dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);           \
1519         dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);           \
1520         dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX);   \
1521         pr_cont("\n");
1522
1523 #define DO_TESTCASE_6_SUCCESS(desc, name)                       \
1524         print_testname(desc);                                   \
1525         dotest(name##_spin, SUCCESS, LOCKTYPE_SPIN);            \
1526         dotest(name##_wlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1527         dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1528         dotest(name##_mutex, SUCCESS, LOCKTYPE_MUTEX);          \
1529         dotest(name##_wsem, SUCCESS, LOCKTYPE_RWSEM);           \
1530         dotest(name##_rsem, SUCCESS, LOCKTYPE_RWSEM);           \
1531         dotest_rt(name##_rtmutex, SUCCESS, LOCKTYPE_RTMUTEX);   \
1532         pr_cont("\n");
1533
1534 /*
1535  * 'read' variant: rlocks must not trigger.
1536  */
1537 #define DO_TESTCASE_6R(desc, name)                              \
1538         print_testname(desc);                                   \
1539         dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);            \
1540         dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);         \
1541         dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1542         dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);          \
1543         dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);           \
1544         dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);           \
1545         dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX);   \
1546         pr_cont("\n");
1547
1548 #define DO_TESTCASE_2I(desc, name, nr)                          \
1549         DO_TESTCASE_1("hard-"desc, name##_hard, nr);            \
1550         DO_TESTCASE_1("soft-"desc, name##_soft, nr);
1551
1552 #define DO_TESTCASE_2IB(desc, name, nr)                         \
1553         DO_TESTCASE_1B("hard-"desc, name##_hard, nr);           \
1554         DO_TESTCASE_1B("soft-"desc, name##_soft, nr);
1555
1556 #define DO_TESTCASE_6I(desc, name, nr)                          \
1557         DO_TESTCASE_3("hard-"desc, name##_hard, nr);            \
1558         DO_TESTCASE_3("soft-"desc, name##_soft, nr);
1559
1560 #define DO_TESTCASE_6IRW(desc, name, nr)                        \
1561         DO_TESTCASE_3RW("hard-"desc, name##_hard, nr);          \
1562         DO_TESTCASE_3RW("soft-"desc, name##_soft, nr);
1563
1564 #define DO_TESTCASE_2x3(desc, name)                             \
1565         DO_TESTCASE_3(desc, name, 12);                          \
1566         DO_TESTCASE_3(desc, name, 21);
1567
1568 #define DO_TESTCASE_2x6(desc, name)                             \
1569         DO_TESTCASE_6I(desc, name, 12);                         \
1570         DO_TESTCASE_6I(desc, name, 21);
1571
1572 #define DO_TESTCASE_6x2(desc, name)                             \
1573         DO_TESTCASE_2I(desc, name, 123);                        \
1574         DO_TESTCASE_2I(desc, name, 132);                        \
1575         DO_TESTCASE_2I(desc, name, 213);                        \
1576         DO_TESTCASE_2I(desc, name, 231);                        \
1577         DO_TESTCASE_2I(desc, name, 312);                        \
1578         DO_TESTCASE_2I(desc, name, 321);
1579
1580 #define DO_TESTCASE_6x2B(desc, name)                            \
1581         DO_TESTCASE_2IB(desc, name, 123);                       \
1582         DO_TESTCASE_2IB(desc, name, 132);                       \
1583         DO_TESTCASE_2IB(desc, name, 213);                       \
1584         DO_TESTCASE_2IB(desc, name, 231);                       \
1585         DO_TESTCASE_2IB(desc, name, 312);                       \
1586         DO_TESTCASE_2IB(desc, name, 321);
1587
1588 #define DO_TESTCASE_6x1RR(desc, name)                           \
1589         DO_TESTCASE_1RR(desc, name, 123);                       \
1590         DO_TESTCASE_1RR(desc, name, 132);                       \
1591         DO_TESTCASE_1RR(desc, name, 213);                       \
1592         DO_TESTCASE_1RR(desc, name, 231);                       \
1593         DO_TESTCASE_1RR(desc, name, 312);                       \
1594         DO_TESTCASE_1RR(desc, name, 321);
1595
1596 #define DO_TESTCASE_6x1RRB(desc, name)                          \
1597         DO_TESTCASE_1RRB(desc, name, 123);                      \
1598         DO_TESTCASE_1RRB(desc, name, 132);                      \
1599         DO_TESTCASE_1RRB(desc, name, 213);                      \
1600         DO_TESTCASE_1RRB(desc, name, 231);                      \
1601         DO_TESTCASE_1RRB(desc, name, 312);                      \
1602         DO_TESTCASE_1RRB(desc, name, 321);
1603
1604 #define DO_TESTCASE_6x6(desc, name)                             \
1605         DO_TESTCASE_6I(desc, name, 123);                        \
1606         DO_TESTCASE_6I(desc, name, 132);                        \
1607         DO_TESTCASE_6I(desc, name, 213);                        \
1608         DO_TESTCASE_6I(desc, name, 231);                        \
1609         DO_TESTCASE_6I(desc, name, 312);                        \
1610         DO_TESTCASE_6I(desc, name, 321);
1611
1612 #define DO_TESTCASE_6x6RW(desc, name)                           \
1613         DO_TESTCASE_6IRW(desc, name, 123);                      \
1614         DO_TESTCASE_6IRW(desc, name, 132);                      \
1615         DO_TESTCASE_6IRW(desc, name, 213);                      \
1616         DO_TESTCASE_6IRW(desc, name, 231);                      \
1617         DO_TESTCASE_6IRW(desc, name, 312);                      \
1618         DO_TESTCASE_6IRW(desc, name, 321);
1619
1620 static void ww_test_fail_acquire(void)
1621 {
1622         int ret;
1623
1624         WWAI(&t);
1625         t.stamp++;
1626
1627         ret = WWL(&o, &t);
1628
1629         if (WARN_ON(!o.ctx) ||
1630             WARN_ON(ret))
1631                 return;
1632
1633         /* No lockdep test, pure API */
1634         ret = WWL(&o, &t);
1635         WARN_ON(ret != -EALREADY);
1636
1637         ret = WWT(&o);
1638         WARN_ON(ret);
1639
1640         t2 = t;
1641         t2.stamp++;
1642         ret = WWL(&o, &t2);
1643         WARN_ON(ret != -EDEADLK);
1644         WWU(&o);
1645
1646         if (WWT(&o))
1647                 WWU(&o);
1648 #ifdef CONFIG_DEBUG_LOCK_ALLOC
1649         else
1650                 DEBUG_LOCKS_WARN_ON(1);
1651 #endif
1652 }
1653
1654 static void ww_test_normal(void)
1655 {
1656         int ret;
1657
1658         WWAI(&t);
1659
1660         /*
1661          * None of the ww_mutex codepaths should be taken in the 'normal'
1662          * mutex calls. The easiest way to verify this is by using the
1663          * normal mutex calls, and making sure o.ctx is unmodified.
1664          */
1665
1666         /* mutex_lock (and indirectly, mutex_lock_nested) */
1667         o.ctx = (void *)~0UL;
1668         mutex_lock(&o.base);
1669         mutex_unlock(&o.base);
1670         WARN_ON(o.ctx != (void *)~0UL);
1671
1672         /* mutex_lock_interruptible (and *_nested) */
1673         o.ctx = (void *)~0UL;
1674         ret = mutex_lock_interruptible(&o.base);
1675         if (!ret)
1676                 mutex_unlock(&o.base);
1677         else
1678                 WARN_ON(1);
1679         WARN_ON(o.ctx != (void *)~0UL);
1680
1681         /* mutex_lock_killable (and *_nested) */
1682         o.ctx = (void *)~0UL;
1683         ret = mutex_lock_killable(&o.base);
1684         if (!ret)
1685                 mutex_unlock(&o.base);
1686         else
1687                 WARN_ON(1);
1688         WARN_ON(o.ctx != (void *)~0UL);
1689
1690         /* trylock, succeeding */
1691         o.ctx = (void *)~0UL;
1692         ret = mutex_trylock(&o.base);
1693         WARN_ON(!ret);
1694         if (ret)
1695                 mutex_unlock(&o.base);
1696         else
1697                 WARN_ON(1);
1698         WARN_ON(o.ctx != (void *)~0UL);
1699
1700         /* trylock, failing */
1701         o.ctx = (void *)~0UL;
1702         mutex_lock(&o.base);
1703         ret = mutex_trylock(&o.base);
1704         WARN_ON(ret);
1705         mutex_unlock(&o.base);
1706         WARN_ON(o.ctx != (void *)~0UL);
1707
1708         /* nest_lock */
1709         o.ctx = (void *)~0UL;
1710         mutex_lock_nest_lock(&o.base, &t);
1711         mutex_unlock(&o.base);
1712         WARN_ON(o.ctx != (void *)~0UL);
1713 }
1714
1715 static void ww_test_two_contexts(void)
1716 {
1717         WWAI(&t);
1718         WWAI(&t2);
1719 }
1720
1721 static void ww_test_diff_class(void)
1722 {
1723         WWAI(&t);
1724 #ifdef CONFIG_DEBUG_MUTEXES
1725         t.ww_class = NULL;
1726 #endif
1727         WWL(&o, &t);
1728 }
1729
1730 static void ww_test_context_done_twice(void)
1731 {
1732         WWAI(&t);
1733         WWAD(&t);
1734         WWAD(&t);
1735         WWAF(&t);
1736 }
1737
1738 static void ww_test_context_unlock_twice(void)
1739 {
1740         WWAI(&t);
1741         WWAD(&t);
1742         WWAF(&t);
1743         WWAF(&t);
1744 }
1745
1746 static void ww_test_context_fini_early(void)
1747 {
1748         WWAI(&t);
1749         WWL(&o, &t);
1750         WWAD(&t);
1751         WWAF(&t);
1752 }
1753
1754 static void ww_test_context_lock_after_done(void)
1755 {
1756         WWAI(&t);
1757         WWAD(&t);
1758         WWL(&o, &t);
1759 }
1760
1761 static void ww_test_object_unlock_twice(void)
1762 {
1763         WWL1(&o);
1764         WWU(&o);
1765         WWU(&o);
1766 }
1767
1768 static void ww_test_object_lock_unbalanced(void)
1769 {
1770         WWAI(&t);
1771         WWL(&o, &t);
1772         t.acquired = 0;
1773         WWU(&o);
1774         WWAF(&t);
1775 }
1776
1777 static void ww_test_object_lock_stale_context(void)
1778 {
1779         WWAI(&t);
1780         o.ctx = &t2;
1781         WWL(&o, &t);
1782 }
1783
1784 static void ww_test_edeadlk_normal(void)
1785 {
1786         int ret;
1787
1788         mutex_lock(&o2.base);
1789         o2.ctx = &t2;
1790         mutex_release(&o2.base.dep_map, _THIS_IP_);
1791
1792         WWAI(&t);
1793         t2 = t;
1794         t2.stamp--;
1795
1796         ret = WWL(&o, &t);
1797         WARN_ON(ret);
1798
1799         ret = WWL(&o2, &t);
1800         WARN_ON(ret != -EDEADLK);
1801
1802         o2.ctx = NULL;
1803         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1804         mutex_unlock(&o2.base);
1805         WWU(&o);
1806
1807         WWL(&o2, &t);
1808 }
1809
1810 static void ww_test_edeadlk_normal_slow(void)
1811 {
1812         int ret;
1813
1814         mutex_lock(&o2.base);
1815         mutex_release(&o2.base.dep_map, _THIS_IP_);
1816         o2.ctx = &t2;
1817
1818         WWAI(&t);
1819         t2 = t;
1820         t2.stamp--;
1821
1822         ret = WWL(&o, &t);
1823         WARN_ON(ret);
1824
1825         ret = WWL(&o2, &t);
1826         WARN_ON(ret != -EDEADLK);
1827
1828         o2.ctx = NULL;
1829         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1830         mutex_unlock(&o2.base);
1831         WWU(&o);
1832
1833         ww_mutex_lock_slow(&o2, &t);
1834 }
1835
1836 static void ww_test_edeadlk_no_unlock(void)
1837 {
1838         int ret;
1839
1840         mutex_lock(&o2.base);
1841         o2.ctx = &t2;
1842         mutex_release(&o2.base.dep_map, _THIS_IP_);
1843
1844         WWAI(&t);
1845         t2 = t;
1846         t2.stamp--;
1847
1848         ret = WWL(&o, &t);
1849         WARN_ON(ret);
1850
1851         ret = WWL(&o2, &t);
1852         WARN_ON(ret != -EDEADLK);
1853
1854         o2.ctx = NULL;
1855         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1856         mutex_unlock(&o2.base);
1857
1858         WWL(&o2, &t);
1859 }
1860
1861 static void ww_test_edeadlk_no_unlock_slow(void)
1862 {
1863         int ret;
1864
1865         mutex_lock(&o2.base);
1866         mutex_release(&o2.base.dep_map, _THIS_IP_);
1867         o2.ctx = &t2;
1868
1869         WWAI(&t);
1870         t2 = t;
1871         t2.stamp--;
1872
1873         ret = WWL(&o, &t);
1874         WARN_ON(ret);
1875
1876         ret = WWL(&o2, &t);
1877         WARN_ON(ret != -EDEADLK);
1878
1879         o2.ctx = NULL;
1880         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1881         mutex_unlock(&o2.base);
1882
1883         ww_mutex_lock_slow(&o2, &t);
1884 }
1885
1886 static void ww_test_edeadlk_acquire_more(void)
1887 {
1888         int ret;
1889
1890         mutex_lock(&o2.base);
1891         mutex_release(&o2.base.dep_map, _THIS_IP_);
1892         o2.ctx = &t2;
1893
1894         WWAI(&t);
1895         t2 = t;
1896         t2.stamp--;
1897
1898         ret = WWL(&o, &t);
1899         WARN_ON(ret);
1900
1901         ret = WWL(&o2, &t);
1902         WARN_ON(ret != -EDEADLK);
1903
1904         ret = WWL(&o3, &t);
1905 }
1906
1907 static void ww_test_edeadlk_acquire_more_slow(void)
1908 {
1909         int ret;
1910
1911         mutex_lock(&o2.base);
1912         mutex_release(&o2.base.dep_map, _THIS_IP_);
1913         o2.ctx = &t2;
1914
1915         WWAI(&t);
1916         t2 = t;
1917         t2.stamp--;
1918
1919         ret = WWL(&o, &t);
1920         WARN_ON(ret);
1921
1922         ret = WWL(&o2, &t);
1923         WARN_ON(ret != -EDEADLK);
1924
1925         ww_mutex_lock_slow(&o3, &t);
1926 }
1927
1928 static void ww_test_edeadlk_acquire_more_edeadlk(void)
1929 {
1930         int ret;
1931
1932         mutex_lock(&o2.base);
1933         mutex_release(&o2.base.dep_map, _THIS_IP_);
1934         o2.ctx = &t2;
1935
1936         mutex_lock(&o3.base);
1937         mutex_release(&o3.base.dep_map, _THIS_IP_);
1938         o3.ctx = &t2;
1939
1940         WWAI(&t);
1941         t2 = t;
1942         t2.stamp--;
1943
1944         ret = WWL(&o, &t);
1945         WARN_ON(ret);
1946
1947         ret = WWL(&o2, &t);
1948         WARN_ON(ret != -EDEADLK);
1949
1950         ret = WWL(&o3, &t);
1951         WARN_ON(ret != -EDEADLK);
1952 }
1953
1954 static void ww_test_edeadlk_acquire_more_edeadlk_slow(void)
1955 {
1956         int ret;
1957
1958         mutex_lock(&o2.base);
1959         mutex_release(&o2.base.dep_map, _THIS_IP_);
1960         o2.ctx = &t2;
1961
1962         mutex_lock(&o3.base);
1963         mutex_release(&o3.base.dep_map, _THIS_IP_);
1964         o3.ctx = &t2;
1965
1966         WWAI(&t);
1967         t2 = t;
1968         t2.stamp--;
1969
1970         ret = WWL(&o, &t);
1971         WARN_ON(ret);
1972
1973         ret = WWL(&o2, &t);
1974         WARN_ON(ret != -EDEADLK);
1975
1976         ww_mutex_lock_slow(&o3, &t);
1977 }
1978
1979 static void ww_test_edeadlk_acquire_wrong(void)
1980 {
1981         int ret;
1982
1983         mutex_lock(&o2.base);
1984         mutex_release(&o2.base.dep_map, _THIS_IP_);
1985         o2.ctx = &t2;
1986
1987         WWAI(&t);
1988         t2 = t;
1989         t2.stamp--;
1990
1991         ret = WWL(&o, &t);
1992         WARN_ON(ret);
1993
1994         ret = WWL(&o2, &t);
1995         WARN_ON(ret != -EDEADLK);
1996         if (!ret)
1997                 WWU(&o2);
1998
1999         WWU(&o);
2000
2001         ret = WWL(&o3, &t);
2002 }
2003
2004 static void ww_test_edeadlk_acquire_wrong_slow(void)
2005 {
2006         int ret;
2007
2008         mutex_lock(&o2.base);
2009         mutex_release(&o2.base.dep_map, _THIS_IP_);
2010         o2.ctx = &t2;
2011
2012         WWAI(&t);
2013         t2 = t;
2014         t2.stamp--;
2015
2016         ret = WWL(&o, &t);
2017         WARN_ON(ret);
2018
2019         ret = WWL(&o2, &t);
2020         WARN_ON(ret != -EDEADLK);
2021         if (!ret)
2022                 WWU(&o2);
2023
2024         WWU(&o);
2025
2026         ww_mutex_lock_slow(&o3, &t);
2027 }
2028
2029 static void ww_test_spin_nest_unlocked(void)
2030 {
2031         spin_lock_nest_lock(&lock_A, &o.base);
2032         U(A);
2033 }
2034
2035 /* This is not a deadlock, because we have X1 to serialize Y1 and Y2 */
2036 static void ww_test_spin_nest_lock(void)
2037 {
2038         spin_lock(&lock_X1);
2039         spin_lock_nest_lock(&lock_Y1, &lock_X1);
2040         spin_lock(&lock_A);
2041         spin_lock_nest_lock(&lock_Y2, &lock_X1);
2042         spin_unlock(&lock_A);
2043         spin_unlock(&lock_Y2);
2044         spin_unlock(&lock_Y1);
2045         spin_unlock(&lock_X1);
2046 }
2047
2048 static void ww_test_unneeded_slow(void)
2049 {
2050         WWAI(&t);
2051
2052         ww_mutex_lock_slow(&o, &t);
2053 }
2054
2055 static void ww_test_context_block(void)
2056 {
2057         int ret;
2058
2059         WWAI(&t);
2060
2061         ret = WWL(&o, &t);
2062         WARN_ON(ret);
2063         WWL1(&o2);
2064 }
2065
2066 static void ww_test_context_try(void)
2067 {
2068         int ret;
2069
2070         WWAI(&t);
2071
2072         ret = WWL(&o, &t);
2073         WARN_ON(ret);
2074
2075         ret = WWT(&o2);
2076         WARN_ON(!ret);
2077         WWU(&o2);
2078         WWU(&o);
2079 }
2080
2081 static void ww_test_context_context(void)
2082 {
2083         int ret;
2084
2085         WWAI(&t);
2086
2087         ret = WWL(&o, &t);
2088         WARN_ON(ret);
2089
2090         ret = WWL(&o2, &t);
2091         WARN_ON(ret);
2092
2093         WWU(&o2);
2094         WWU(&o);
2095 }
2096
2097 static void ww_test_try_block(void)
2098 {
2099         bool ret;
2100
2101         ret = WWT(&o);
2102         WARN_ON(!ret);
2103
2104         WWL1(&o2);
2105         WWU(&o2);
2106         WWU(&o);
2107 }
2108
2109 static void ww_test_try_try(void)
2110 {
2111         bool ret;
2112
2113         ret = WWT(&o);
2114         WARN_ON(!ret);
2115         ret = WWT(&o2);
2116         WARN_ON(!ret);
2117         WWU(&o2);
2118         WWU(&o);
2119 }
2120
2121 static void ww_test_try_context(void)
2122 {
2123         int ret;
2124
2125         ret = WWT(&o);
2126         WARN_ON(!ret);
2127
2128         WWAI(&t);
2129
2130         ret = WWL(&o2, &t);
2131         WARN_ON(ret);
2132 }
2133
2134 static void ww_test_block_block(void)
2135 {
2136         WWL1(&o);
2137         WWL1(&o2);
2138 }
2139
2140 static void ww_test_block_try(void)
2141 {
2142         bool ret;
2143
2144         WWL1(&o);
2145         ret = WWT(&o2);
2146         WARN_ON(!ret);
2147 }
2148
2149 static void ww_test_block_context(void)
2150 {
2151         int ret;
2152
2153         WWL1(&o);
2154         WWAI(&t);
2155
2156         ret = WWL(&o2, &t);
2157         WARN_ON(ret);
2158 }
2159
2160 static void ww_test_spin_block(void)
2161 {
2162         L(A);
2163         U(A);
2164
2165         WWL1(&o);
2166         L(A);
2167         U(A);
2168         WWU(&o);
2169
2170         L(A);
2171         WWL1(&o);
2172         WWU(&o);
2173         U(A);
2174 }
2175
2176 static void ww_test_spin_try(void)
2177 {
2178         bool ret;
2179
2180         L(A);
2181         U(A);
2182
2183         ret = WWT(&o);
2184         WARN_ON(!ret);
2185         L(A);
2186         U(A);
2187         WWU(&o);
2188
2189         L(A);
2190         ret = WWT(&o);
2191         WARN_ON(!ret);
2192         WWU(&o);
2193         U(A);
2194 }
2195
2196 static void ww_test_spin_context(void)
2197 {
2198         int ret;
2199
2200         L(A);
2201         U(A);
2202
2203         WWAI(&t);
2204
2205         ret = WWL(&o, &t);
2206         WARN_ON(ret);
2207         L(A);
2208         U(A);
2209         WWU(&o);
2210
2211         L(A);
2212         ret = WWL(&o, &t);
2213         WARN_ON(ret);
2214         WWU(&o);
2215         U(A);
2216 }
2217
2218 static void ww_tests(void)
2219 {
2220         printk("  --------------------------------------------------------------------------\n");
2221         printk("  | Wound/wait tests |\n");
2222         printk("  ---------------------\n");
2223
2224         print_testname("ww api failures");
2225         dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW);
2226         dotest(ww_test_normal, SUCCESS, LOCKTYPE_WW);
2227         dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW);
2228         pr_cont("\n");
2229
2230         print_testname("ww contexts mixing");
2231         dotest(ww_test_two_contexts, FAILURE, LOCKTYPE_WW);
2232         dotest(ww_test_diff_class, FAILURE, LOCKTYPE_WW);
2233         pr_cont("\n");
2234
2235         print_testname("finishing ww context");
2236         dotest(ww_test_context_done_twice, FAILURE, LOCKTYPE_WW);
2237         dotest(ww_test_context_unlock_twice, FAILURE, LOCKTYPE_WW);
2238         dotest(ww_test_context_fini_early, FAILURE, LOCKTYPE_WW);
2239         dotest(ww_test_context_lock_after_done, FAILURE, LOCKTYPE_WW);
2240         pr_cont("\n");
2241
2242         print_testname("locking mismatches");
2243         dotest(ww_test_object_unlock_twice, FAILURE, LOCKTYPE_WW);
2244         dotest(ww_test_object_lock_unbalanced, FAILURE, LOCKTYPE_WW);
2245         dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW);
2246         pr_cont("\n");
2247
2248         print_testname("EDEADLK handling");
2249         dotest(ww_test_edeadlk_normal, SUCCESS, LOCKTYPE_WW);
2250         dotest(ww_test_edeadlk_normal_slow, SUCCESS, LOCKTYPE_WW);
2251         dotest(ww_test_edeadlk_no_unlock, FAILURE, LOCKTYPE_WW);
2252         dotest(ww_test_edeadlk_no_unlock_slow, FAILURE, LOCKTYPE_WW);
2253         dotest(ww_test_edeadlk_acquire_more, FAILURE, LOCKTYPE_WW);
2254         dotest(ww_test_edeadlk_acquire_more_slow, FAILURE, LOCKTYPE_WW);
2255         dotest(ww_test_edeadlk_acquire_more_edeadlk, FAILURE, LOCKTYPE_WW);
2256         dotest(ww_test_edeadlk_acquire_more_edeadlk_slow, FAILURE, LOCKTYPE_WW);
2257         dotest(ww_test_edeadlk_acquire_wrong, FAILURE, LOCKTYPE_WW);
2258         dotest(ww_test_edeadlk_acquire_wrong_slow, FAILURE, LOCKTYPE_WW);
2259         pr_cont("\n");
2260
2261         print_testname("spinlock nest unlocked");
2262         dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW);
2263         pr_cont("\n");
2264
2265         print_testname("spinlock nest test");
2266         dotest(ww_test_spin_nest_lock, SUCCESS, LOCKTYPE_WW);
2267         pr_cont("\n");
2268
2269         printk("  -----------------------------------------------------\n");
2270         printk("                                 |block | try  |context|\n");
2271         printk("  -----------------------------------------------------\n");
2272
2273         print_testname("context");
2274         dotest(ww_test_context_block, FAILURE, LOCKTYPE_WW);
2275         dotest(ww_test_context_try, SUCCESS, LOCKTYPE_WW);
2276         dotest(ww_test_context_context, SUCCESS, LOCKTYPE_WW);
2277         pr_cont("\n");
2278
2279         print_testname("try");
2280         dotest(ww_test_try_block, FAILURE, LOCKTYPE_WW);
2281         dotest(ww_test_try_try, SUCCESS, LOCKTYPE_WW);
2282         dotest(ww_test_try_context, FAILURE, LOCKTYPE_WW);
2283         pr_cont("\n");
2284
2285         print_testname("block");
2286         dotest(ww_test_block_block, FAILURE, LOCKTYPE_WW);
2287         dotest(ww_test_block_try, SUCCESS, LOCKTYPE_WW);
2288         dotest(ww_test_block_context, FAILURE, LOCKTYPE_WW);
2289         pr_cont("\n");
2290
2291         print_testname("spinlock");
2292         dotest(ww_test_spin_block, FAILURE, LOCKTYPE_WW);
2293         dotest(ww_test_spin_try, SUCCESS, LOCKTYPE_WW);
2294         dotest(ww_test_spin_context, FAILURE, LOCKTYPE_WW);
2295         pr_cont("\n");
2296 }
2297
2298
2299 /*
2300  * <in hardirq handler>
2301  * read_lock(&A);
2302  *                      <hardirq disable>
2303  *                      spin_lock(&B);
2304  * spin_lock(&B);
2305  *                      read_lock(&A);
2306  *
2307  * is a deadlock.
2308  */
2309 static void queued_read_lock_hardirq_RE_Er(void)
2310 {
2311         HARDIRQ_ENTER();
2312         read_lock(&rwlock_A);
2313         LOCK(B);
2314         UNLOCK(B);
2315         read_unlock(&rwlock_A);
2316         HARDIRQ_EXIT();
2317
2318         HARDIRQ_DISABLE();
2319         LOCK(B);
2320         read_lock(&rwlock_A);
2321         read_unlock(&rwlock_A);
2322         UNLOCK(B);
2323         HARDIRQ_ENABLE();
2324 }
2325
2326 /*
2327  * <in hardirq handler>
2328  * spin_lock(&B);
2329  *                      <hardirq disable>
2330  *                      read_lock(&A);
2331  * read_lock(&A);
2332  *                      spin_lock(&B);
2333  *
2334  * is not a deadlock.
2335  */
2336 static void queued_read_lock_hardirq_ER_rE(void)
2337 {
2338         HARDIRQ_ENTER();
2339         LOCK(B);
2340         read_lock(&rwlock_A);
2341         read_unlock(&rwlock_A);
2342         UNLOCK(B);
2343         HARDIRQ_EXIT();
2344
2345         HARDIRQ_DISABLE();
2346         read_lock(&rwlock_A);
2347         LOCK(B);
2348         UNLOCK(B);
2349         read_unlock(&rwlock_A);
2350         HARDIRQ_ENABLE();
2351 }
2352
2353 /*
2354  * <hardirq disable>
2355  * spin_lock(&B);
2356  *                      read_lock(&A);
2357  *                      <in hardirq handler>
2358  *                      spin_lock(&B);
2359  * read_lock(&A);
2360  *
2361  * is a deadlock. Because the two read_lock()s are both non-recursive readers.
2362  */
2363 static void queued_read_lock_hardirq_inversion(void)
2364 {
2365
2366         HARDIRQ_ENTER();
2367         LOCK(B);
2368         UNLOCK(B);
2369         HARDIRQ_EXIT();
2370
2371         HARDIRQ_DISABLE();
2372         LOCK(B);
2373         read_lock(&rwlock_A);
2374         read_unlock(&rwlock_A);
2375         UNLOCK(B);
2376         HARDIRQ_ENABLE();
2377
2378         read_lock(&rwlock_A);
2379         read_unlock(&rwlock_A);
2380 }
2381
2382 static void queued_read_lock_tests(void)
2383 {
2384         printk("  --------------------------------------------------------------------------\n");
2385         printk("  | queued read lock tests |\n");
2386         printk("  ---------------------------\n");
2387         print_testname("hardirq read-lock/lock-read");
2388         dotest(queued_read_lock_hardirq_RE_Er, FAILURE, LOCKTYPE_RWLOCK);
2389         pr_cont("\n");
2390
2391         print_testname("hardirq lock-read/read-lock");
2392         dotest(queued_read_lock_hardirq_ER_rE, SUCCESS, LOCKTYPE_RWLOCK);
2393         pr_cont("\n");
2394
2395         print_testname("hardirq inversion");
2396         dotest(queued_read_lock_hardirq_inversion, FAILURE, LOCKTYPE_RWLOCK);
2397         pr_cont("\n");
2398 }
2399
2400 static void fs_reclaim_correct_nesting(void)
2401 {
2402         fs_reclaim_acquire(GFP_KERNEL);
2403         might_alloc(GFP_NOFS);
2404         fs_reclaim_release(GFP_KERNEL);
2405 }
2406
2407 static void fs_reclaim_wrong_nesting(void)
2408 {
2409         fs_reclaim_acquire(GFP_KERNEL);
2410         might_alloc(GFP_KERNEL);
2411         fs_reclaim_release(GFP_KERNEL);
2412 }
2413
2414 static void fs_reclaim_protected_nesting(void)
2415 {
2416         unsigned int flags;
2417
2418         fs_reclaim_acquire(GFP_KERNEL);
2419         flags = memalloc_nofs_save();
2420         might_alloc(GFP_KERNEL);
2421         memalloc_nofs_restore(flags);
2422         fs_reclaim_release(GFP_KERNEL);
2423 }
2424
2425 static void fs_reclaim_tests(void)
2426 {
2427         printk("  --------------------\n");
2428         printk("  | fs_reclaim tests |\n");
2429         printk("  --------------------\n");
2430
2431         print_testname("correct nesting");
2432         dotest(fs_reclaim_correct_nesting, SUCCESS, 0);
2433         pr_cont("\n");
2434
2435         print_testname("wrong nesting");
2436         dotest(fs_reclaim_wrong_nesting, FAILURE, 0);
2437         pr_cont("\n");
2438
2439         print_testname("protected nesting");
2440         dotest(fs_reclaim_protected_nesting, SUCCESS, 0);
2441         pr_cont("\n");
2442 }
2443
2444 #define __guard(cleanup) __maybe_unused __attribute__((__cleanup__(cleanup)))
2445
2446 static void hardirq_exit(int *_)
2447 {
2448         HARDIRQ_EXIT();
2449 }
2450
2451 #define HARDIRQ_CONTEXT(name, ...)                                      \
2452         int hardirq_guard_##name __guard(hardirq_exit);                 \
2453         HARDIRQ_ENTER();
2454
2455 #define NOTTHREADED_HARDIRQ_CONTEXT(name, ...)                          \
2456         int notthreaded_hardirq_guard_##name __guard(hardirq_exit);     \
2457         local_irq_disable();                                            \
2458         __irq_enter();                                                  \
2459         WARN_ON(!in_irq());
2460
2461 static void softirq_exit(int *_)
2462 {
2463         SOFTIRQ_EXIT();
2464 }
2465
2466 #define SOFTIRQ_CONTEXT(name, ...)                              \
2467         int softirq_guard_##name __guard(softirq_exit);         \
2468         SOFTIRQ_ENTER();
2469
2470 static void rcu_exit(int *_)
2471 {
2472         rcu_read_unlock();
2473 }
2474
2475 #define RCU_CONTEXT(name, ...)                                  \
2476         int rcu_guard_##name __guard(rcu_exit);                 \
2477         rcu_read_lock();
2478
2479 static void rcu_bh_exit(int *_)
2480 {
2481         rcu_read_unlock_bh();
2482 }
2483
2484 #define RCU_BH_CONTEXT(name, ...)                               \
2485         int rcu_bh_guard_##name __guard(rcu_bh_exit);           \
2486         rcu_read_lock_bh();
2487
2488 static void rcu_sched_exit(int *_)
2489 {
2490         rcu_read_unlock_sched();
2491 }
2492
2493 #define RCU_SCHED_CONTEXT(name, ...)                            \
2494         int rcu_sched_guard_##name __guard(rcu_sched_exit);     \
2495         rcu_read_lock_sched();
2496
2497 static void raw_spinlock_exit(raw_spinlock_t **lock)
2498 {
2499         raw_spin_unlock(*lock);
2500 }
2501
2502 #define RAW_SPINLOCK_CONTEXT(name, lock)                                                \
2503         raw_spinlock_t *raw_spinlock_guard_##name __guard(raw_spinlock_exit) = &(lock); \
2504         raw_spin_lock(&(lock));
2505
2506 static void spinlock_exit(spinlock_t **lock)
2507 {
2508         spin_unlock(*lock);
2509 }
2510
2511 #define SPINLOCK_CONTEXT(name, lock)                                            \
2512         spinlock_t *spinlock_guard_##name __guard(spinlock_exit) = &(lock);     \
2513         spin_lock(&(lock));
2514
2515 static void mutex_exit(struct mutex **lock)
2516 {
2517         mutex_unlock(*lock);
2518 }
2519
2520 #define MUTEX_CONTEXT(name, lock)                                       \
2521         struct mutex *mutex_guard_##name __guard(mutex_exit) = &(lock); \
2522         mutex_lock(&(lock));
2523
2524 #define GENERATE_2_CONTEXT_TESTCASE(outer, outer_lock, inner, inner_lock)       \
2525                                                                                 \
2526 static void __maybe_unused inner##_in_##outer(void)                             \
2527 {                                                                               \
2528         outer##_CONTEXT(_, outer_lock);                                         \
2529         {                                                                       \
2530                 inner##_CONTEXT(_, inner_lock);                                 \
2531         }                                                                       \
2532 }
2533
2534 /*
2535  * wait contexts (considering PREEMPT_RT)
2536  *
2537  * o: inner is allowed in outer
2538  * x: inner is disallowed in outer
2539  *
2540  *       \  inner |  RCU  | RAW_SPIN | SPIN | MUTEX
2541  * outer  \       |       |          |      |
2542  * ---------------+-------+----------+------+-------
2543  * HARDIRQ        |   o   |    o     |  o   |  x
2544  * ---------------+-------+----------+------+-------
2545  * NOTTHREADED_IRQ|   o   |    o     |  x   |  x
2546  * ---------------+-------+----------+------+-------
2547  * SOFTIRQ        |   o   |    o     |  o   |  x
2548  * ---------------+-------+----------+------+-------
2549  * RCU            |   o   |    o     |  o   |  x
2550  * ---------------+-------+----------+------+-------
2551  * RCU_BH         |   o   |    o     |  o   |  x
2552  * ---------------+-------+----------+------+-------
2553  * RCU_SCHED      |   o   |    o     |  x   |  x
2554  * ---------------+-------+----------+------+-------
2555  * RAW_SPIN       |   o   |    o     |  x   |  x
2556  * ---------------+-------+----------+------+-------
2557  * SPIN           |   o   |    o     |  o   |  x
2558  * ---------------+-------+----------+------+-------
2559  * MUTEX          |   o   |    o     |  o   |  o
2560  * ---------------+-------+----------+------+-------
2561  */
2562
2563 #define GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(inner, inner_lock)            \
2564 GENERATE_2_CONTEXT_TESTCASE(HARDIRQ, , inner, inner_lock)                       \
2565 GENERATE_2_CONTEXT_TESTCASE(NOTTHREADED_HARDIRQ, , inner, inner_lock)           \
2566 GENERATE_2_CONTEXT_TESTCASE(SOFTIRQ, , inner, inner_lock)                       \
2567 GENERATE_2_CONTEXT_TESTCASE(RCU, , inner, inner_lock)                           \
2568 GENERATE_2_CONTEXT_TESTCASE(RCU_BH, , inner, inner_lock)                        \
2569 GENERATE_2_CONTEXT_TESTCASE(RCU_SCHED, , inner, inner_lock)                     \
2570 GENERATE_2_CONTEXT_TESTCASE(RAW_SPINLOCK, raw_lock_A, inner, inner_lock)        \
2571 GENERATE_2_CONTEXT_TESTCASE(SPINLOCK, lock_A, inner, inner_lock)                \
2572 GENERATE_2_CONTEXT_TESTCASE(MUTEX, mutex_A, inner, inner_lock)
2573
2574 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RCU, )
2575 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RAW_SPINLOCK, raw_lock_B)
2576 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(SPINLOCK, lock_B)
2577 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(MUTEX, mutex_B)
2578
2579 /* the outer context allows all kinds of preemption */
2580 #define DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(outer)                    \
2581         dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK);               \
2582         dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);        \
2583         dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);            \
2584         dotest(MUTEX_in_##outer, SUCCESS, LOCKTYPE_MUTEX);              \
2585
2586 /*
2587  * the outer context only allows the preemption introduced by spinlock_t (which
2588  * is a sleepable lock for PREEMPT_RT)
2589  */
2590 #define DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(outer)            \
2591         dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK);               \
2592         dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);        \
2593         dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);            \
2594         dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX);              \
2595
2596 /* the outer doesn't allows any kind of preemption */
2597 #define DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(outer)                        \
2598         dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK);               \
2599         dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);        \
2600         dotest(SPINLOCK_in_##outer, FAILURE, LOCKTYPE_SPIN);            \
2601         dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX);              \
2602
2603 static void wait_context_tests(void)
2604 {
2605         printk("  --------------------------------------------------------------------------\n");
2606         printk("  | wait context tests |\n");
2607         printk("  --------------------------------------------------------------------------\n");
2608         printk("                                 | rcu  | raw  | spin |mutex |\n");
2609         printk("  --------------------------------------------------------------------------\n");
2610         print_testname("in hardirq context");
2611         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(HARDIRQ);
2612         pr_cont("\n");
2613
2614         print_testname("in hardirq context (not threaded)");
2615         DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(NOTTHREADED_HARDIRQ);
2616         pr_cont("\n");
2617
2618         print_testname("in softirq context");
2619         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SOFTIRQ);
2620         pr_cont("\n");
2621
2622         print_testname("in RCU context");
2623         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU);
2624         pr_cont("\n");
2625
2626         print_testname("in RCU-bh context");
2627         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU_BH);
2628         pr_cont("\n");
2629
2630         print_testname("in RCU-sched context");
2631         DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RCU_SCHED);
2632         pr_cont("\n");
2633
2634         print_testname("in RAW_SPINLOCK context");
2635         DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RAW_SPINLOCK);
2636         pr_cont("\n");
2637
2638         print_testname("in SPINLOCK context");
2639         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SPINLOCK);
2640         pr_cont("\n");
2641
2642         print_testname("in MUTEX context");
2643         DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(MUTEX);
2644         pr_cont("\n");
2645 }
2646
2647 static void local_lock_2(void)
2648 {
2649         local_lock_acquire(&local_A);   /* IRQ-ON */
2650         local_lock_release(&local_A);
2651
2652         HARDIRQ_ENTER();
2653         spin_lock(&lock_A);             /* IN-IRQ */
2654         spin_unlock(&lock_A);
2655         HARDIRQ_EXIT()
2656
2657         HARDIRQ_DISABLE();
2658         spin_lock(&lock_A);
2659         local_lock_acquire(&local_A);   /* IN-IRQ <-> IRQ-ON cycle, false */
2660         local_lock_release(&local_A);
2661         spin_unlock(&lock_A);
2662         HARDIRQ_ENABLE();
2663 }
2664
2665 static void local_lock_3A(void)
2666 {
2667         local_lock_acquire(&local_A);   /* IRQ-ON */
2668         spin_lock(&lock_B);             /* IRQ-ON */
2669         spin_unlock(&lock_B);
2670         local_lock_release(&local_A);
2671
2672         HARDIRQ_ENTER();
2673         spin_lock(&lock_A);             /* IN-IRQ */
2674         spin_unlock(&lock_A);
2675         HARDIRQ_EXIT()
2676
2677         HARDIRQ_DISABLE();
2678         spin_lock(&lock_A);
2679         local_lock_acquire(&local_A);   /* IN-IRQ <-> IRQ-ON cycle only if we count local_lock(), false */
2680         local_lock_release(&local_A);
2681         spin_unlock(&lock_A);
2682         HARDIRQ_ENABLE();
2683 }
2684
2685 static void local_lock_3B(void)
2686 {
2687         local_lock_acquire(&local_A);   /* IRQ-ON */
2688         spin_lock(&lock_B);             /* IRQ-ON */
2689         spin_unlock(&lock_B);
2690         local_lock_release(&local_A);
2691
2692         HARDIRQ_ENTER();
2693         spin_lock(&lock_A);             /* IN-IRQ */
2694         spin_unlock(&lock_A);
2695         HARDIRQ_EXIT()
2696
2697         HARDIRQ_DISABLE();
2698         spin_lock(&lock_A);
2699         local_lock_acquire(&local_A);   /* IN-IRQ <-> IRQ-ON cycle only if we count local_lock(), false */
2700         local_lock_release(&local_A);
2701         spin_unlock(&lock_A);
2702         HARDIRQ_ENABLE();
2703
2704         HARDIRQ_DISABLE();
2705         spin_lock(&lock_A);
2706         spin_lock(&lock_B);             /* IN-IRQ <-> IRQ-ON cycle, true */
2707         spin_unlock(&lock_B);
2708         spin_unlock(&lock_A);
2709         HARDIRQ_DISABLE();
2710
2711 }
2712
2713 static void local_lock_tests(void)
2714 {
2715         printk("  --------------------------------------------------------------------------\n");
2716         printk("  | local_lock tests |\n");
2717         printk("  ---------------------\n");
2718
2719         print_testname("local_lock inversion  2");
2720         dotest(local_lock_2, SUCCESS, LOCKTYPE_LL);
2721         pr_cont("\n");
2722
2723         print_testname("local_lock inversion 3A");
2724         dotest(local_lock_3A, SUCCESS, LOCKTYPE_LL);
2725         pr_cont("\n");
2726
2727         print_testname("local_lock inversion 3B");
2728         dotest(local_lock_3B, FAILURE, LOCKTYPE_LL);
2729         pr_cont("\n");
2730 }
2731
2732 static void hardirq_deadlock_softirq_not_deadlock(void)
2733 {
2734         /* mutex_A is hardirq-unsafe and softirq-unsafe */
2735         /* mutex_A -> lock_C */
2736         mutex_lock(&mutex_A);
2737         HARDIRQ_DISABLE();
2738         spin_lock(&lock_C);
2739         spin_unlock(&lock_C);
2740         HARDIRQ_ENABLE();
2741         mutex_unlock(&mutex_A);
2742
2743         /* lock_A is hardirq-safe */
2744         HARDIRQ_ENTER();
2745         spin_lock(&lock_A);
2746         spin_unlock(&lock_A);
2747         HARDIRQ_EXIT();
2748
2749         /* lock_A -> lock_B */
2750         HARDIRQ_DISABLE();
2751         spin_lock(&lock_A);
2752         spin_lock(&lock_B);
2753         spin_unlock(&lock_B);
2754         spin_unlock(&lock_A);
2755         HARDIRQ_ENABLE();
2756
2757         /* lock_B -> lock_C */
2758         HARDIRQ_DISABLE();
2759         spin_lock(&lock_B);
2760         spin_lock(&lock_C);
2761         spin_unlock(&lock_C);
2762         spin_unlock(&lock_B);
2763         HARDIRQ_ENABLE();
2764
2765         /* lock_D is softirq-safe */
2766         SOFTIRQ_ENTER();
2767         spin_lock(&lock_D);
2768         spin_unlock(&lock_D);
2769         SOFTIRQ_EXIT();
2770
2771         /* And lock_D is hardirq-unsafe */
2772         SOFTIRQ_DISABLE();
2773         spin_lock(&lock_D);
2774         spin_unlock(&lock_D);
2775         SOFTIRQ_ENABLE();
2776
2777         /*
2778          * mutex_A -> lock_C -> lock_D is softirq-unsafe -> softirq-safe, not
2779          * deadlock.
2780          *
2781          * lock_A -> lock_B -> lock_C -> lock_D is hardirq-safe ->
2782          * hardirq-unsafe, deadlock.
2783          */
2784         HARDIRQ_DISABLE();
2785         spin_lock(&lock_C);
2786         spin_lock(&lock_D);
2787         spin_unlock(&lock_D);
2788         spin_unlock(&lock_C);
2789         HARDIRQ_ENABLE();
2790 }
2791
2792 void locking_selftest(void)
2793 {
2794         /*
2795          * Got a locking failure before the selftest ran?
2796          */
2797         if (!debug_locks) {
2798                 printk("----------------------------------\n");
2799                 printk("| Locking API testsuite disabled |\n");
2800                 printk("----------------------------------\n");
2801                 return;
2802         }
2803
2804         /*
2805          * treats read_lock() as recursive read locks for testing purpose
2806          */
2807         force_read_lock_recursive = 1;
2808
2809         /*
2810          * Run the testsuite:
2811          */
2812         printk("------------------------\n");
2813         printk("| Locking API testsuite:\n");
2814         printk("----------------------------------------------------------------------------\n");
2815         printk("                                 | spin |wlock |rlock |mutex | wsem | rsem |\n");
2816         printk("  --------------------------------------------------------------------------\n");
2817
2818         init_shared_classes();
2819         lockdep_set_selftest_task(current);
2820
2821         DO_TESTCASE_6R("A-A deadlock", AA);
2822         DO_TESTCASE_6R("A-B-B-A deadlock", ABBA);
2823         DO_TESTCASE_6R("A-B-B-C-C-A deadlock", ABBCCA);
2824         DO_TESTCASE_6R("A-B-C-A-B-C deadlock", ABCABC);
2825         DO_TESTCASE_6R("A-B-B-C-C-D-D-A deadlock", ABBCCDDA);
2826         DO_TESTCASE_6R("A-B-C-D-B-D-D-A deadlock", ABCDBDDA);
2827         DO_TESTCASE_6R("A-B-C-D-B-C-D-A deadlock", ABCDBCDA);
2828         DO_TESTCASE_6("double unlock", double_unlock);
2829         DO_TESTCASE_6("initialize held", init_held);
2830
2831         printk("  --------------------------------------------------------------------------\n");
2832         print_testname("recursive read-lock");
2833         pr_cont("             |");
2834         dotest(rlock_AA1, SUCCESS, LOCKTYPE_RWLOCK);
2835         pr_cont("             |");
2836         dotest(rsem_AA1, FAILURE, LOCKTYPE_RWSEM);
2837         pr_cont("\n");
2838
2839         print_testname("recursive read-lock #2");
2840         pr_cont("             |");
2841         dotest(rlock_AA1B, SUCCESS, LOCKTYPE_RWLOCK);
2842         pr_cont("             |");
2843         dotest(rsem_AA1B, FAILURE, LOCKTYPE_RWSEM);
2844         pr_cont("\n");
2845
2846         print_testname("mixed read-write-lock");
2847         pr_cont("             |");
2848         dotest(rlock_AA2, FAILURE, LOCKTYPE_RWLOCK);
2849         pr_cont("             |");
2850         dotest(rsem_AA2, FAILURE, LOCKTYPE_RWSEM);
2851         pr_cont("\n");
2852
2853         print_testname("mixed write-read-lock");
2854         pr_cont("             |");
2855         dotest(rlock_AA3, FAILURE, LOCKTYPE_RWLOCK);
2856         pr_cont("             |");
2857         dotest(rsem_AA3, FAILURE, LOCKTYPE_RWSEM);
2858         pr_cont("\n");
2859
2860         print_testname("mixed read-lock/lock-write ABBA");
2861         pr_cont("             |");
2862         dotest(rlock_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2863         pr_cont("             |");
2864         dotest(rwsem_ABBA1, FAILURE, LOCKTYPE_RWSEM);
2865
2866         print_testname("mixed read-lock/lock-read ABBA");
2867         pr_cont("             |");
2868         dotest(rlock_ABBA2, SUCCESS, LOCKTYPE_RWLOCK);
2869         pr_cont("             |");
2870         dotest(rwsem_ABBA2, FAILURE, LOCKTYPE_RWSEM);
2871
2872         print_testname("mixed write-lock/lock-write ABBA");
2873         pr_cont("             |");
2874         dotest(rlock_ABBA3, FAILURE, LOCKTYPE_RWLOCK);
2875         pr_cont("             |");
2876         dotest(rwsem_ABBA3, FAILURE, LOCKTYPE_RWSEM);
2877
2878         print_testname("chain cached mixed R-L/L-W ABBA");
2879         pr_cont("             |");
2880         dotest(rlock_chaincache_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2881
2882         DO_TESTCASE_6x1RRB("rlock W1R2/W2R3/W3R1", W1R2_W2R3_W3R1);
2883         DO_TESTCASE_6x1RRB("rlock W1W2/R2R3/W3R1", W1W2_R2R3_W3R1);
2884         DO_TESTCASE_6x1RR("rlock W1W2/R2R3/R3W1", W1W2_R2R3_R3W1);
2885         DO_TESTCASE_6x1RR("rlock W1R2/R2R3/W3W1", W1R2_R2R3_W3W1);
2886
2887         printk("  --------------------------------------------------------------------------\n");
2888
2889         /*
2890          * irq-context testcases:
2891          */
2892         DO_TESTCASE_2x6("irqs-on + irq-safe-A", irqsafe1);
2893         DO_TESTCASE_2x3("sirq-safe-A => hirqs-on", irqsafe2A);
2894         DO_TESTCASE_2x6("safe-A + irqs-on", irqsafe2B);
2895         DO_TESTCASE_6x6("safe-A + unsafe-B #1", irqsafe3);
2896         DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4);
2897         DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion);
2898
2899         DO_TESTCASE_6x2x2RW("irq read-recursion", irq_read_recursion);
2900         DO_TESTCASE_6x2x2RW("irq read-recursion #2", irq_read_recursion2);
2901         DO_TESTCASE_6x2x2RW("irq read-recursion #3", irq_read_recursion3);
2902
2903         ww_tests();
2904
2905         force_read_lock_recursive = 0;
2906         /*
2907          * queued_read_lock() specific test cases can be put here
2908          */
2909         if (IS_ENABLED(CONFIG_QUEUED_RWLOCKS))
2910                 queued_read_lock_tests();
2911
2912         fs_reclaim_tests();
2913
2914         /* Wait context test cases that are specific for RAW_LOCK_NESTING */
2915         if (IS_ENABLED(CONFIG_PROVE_RAW_LOCK_NESTING))
2916                 wait_context_tests();
2917
2918         local_lock_tests();
2919
2920         print_testname("hardirq_unsafe_softirq_safe");
2921         dotest(hardirq_deadlock_softirq_not_deadlock, FAILURE, LOCKTYPE_SPECIAL);
2922         pr_cont("\n");
2923
2924         if (unexpected_testcase_failures) {
2925                 printk("-----------------------------------------------------------------\n");
2926                 debug_locks = 0;
2927                 printk("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n",
2928                         unexpected_testcase_failures, testcase_total);
2929                 printk("-----------------------------------------------------------------\n");
2930         } else if (expected_testcase_failures && testcase_successes) {
2931                 printk("--------------------------------------------------------\n");
2932                 printk("%3d out of %3d testcases failed, as expected. |\n",
2933                         expected_testcase_failures, testcase_total);
2934                 printk("----------------------------------------------------\n");
2935                 debug_locks = 1;
2936         } else if (expected_testcase_failures && !testcase_successes) {
2937                 printk("--------------------------------------------------------\n");
2938                 printk("All %3d testcases failed, as expected. |\n",
2939                         expected_testcase_failures);
2940                 printk("----------------------------------------\n");
2941                 debug_locks = 1;
2942         } else {
2943                 printk("-------------------------------------------------------\n");
2944                 printk("Good, all %3d testcases passed! |\n",
2945                         testcase_successes);
2946                 printk("---------------------------------\n");
2947                 debug_locks = 1;
2948         }
2949         lockdep_set_selftest_task(NULL);
2950         debug_locks_silent = 0;
2951 }