kernel/locking/locktorture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Module-based torture test facility for locking
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *          Davidlohr Bueso <dave@stgolabs.net>
   9  *      Based on kernel/rcu/torture.c.
  10  */
  11
  12 #define pr_fmt(fmt) fmt
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/kthread.h>
  17 #include <linux/sched/rt.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rwsem.h>
  21 #include <linux/smp.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/sched.h>
  24 #include <uapi/linux/sched/types.h>
  25 #include <linux/rtmutex.h>
  26 #include <linux/atomic.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/delay.h>
  29 #include <linux/slab.h>
  30 #include <linux/percpu-rwsem.h>
  31 #include <linux/torture.h>
  32
  33 MODULE_LICENSE("GPL");
  34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  35
  36 torture_param(int, nwriters_stress, -1,
  37              "Number of write-locking stress-test threads");
  38 torture_param(int, nreaders_stress, -1,
  39              "Number of read-locking stress-test threads");
  40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  41 torture_param(int, onoff_interval, 0,
  42              "Time between CPU hotplugs (s), 0=disable");
  43 torture_param(int, shuffle_interval, 3,
  44              "Number of jiffies between shuffles, 0=disable");
  45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
  46 torture_param(int, stat_interval, 60,
  47              "Number of seconds between stats printk()s");
  48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
  49 torture_param(int, verbose, 1,
  50              "Enable verbose debugging printk()s");
  51
  52 static char *torture_type = "spin_lock";
  53 module_param(torture_type, charp, 0444);
  54 MODULE_PARM_DESC(torture_type,
  55                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
  56
  57 static struct task_struct *stats_task;
  58 static struct task_struct **writer_tasks;
  59 static struct task_struct **reader_tasks;
  60
  61 static bool lock_is_write_held;
  62 static bool lock_is_read_held;
  63
  64 struct lock_stress_stats {
  65         long n_lock_fail;
  66         long n_lock_acquired;
  67 };
  68
  69 /* Forward reference. */
  70 static void lock_torture_cleanup(void);
  71
  72 /*
  73  * Operations vector for selecting different types of tests.
  74  */
  75 struct lock_torture_ops {
  76         void (*init)(void);
  77         int (*writelock)(void);
  78         void (*write_delay)(struct torture_random_state *trsp);
  79         void (*task_boost)(struct torture_random_state *trsp);
  80         void (*writeunlock)(void);
  81         int (*readlock)(void);
  82         void (*read_delay)(struct torture_random_state *trsp);
  83         void (*readunlock)(void);
  84
  85         unsigned long flags; /* for irq spinlocks */
  86         const char *name;
  87 };
  88
  89 struct lock_torture_cxt {
  90         int nrealwriters_stress;
  91         int nrealreaders_stress;
  92         bool debug_lock;
  93         atomic_t n_lock_torture_errors;
  94         struct lock_torture_ops *cur_ops;
  95         struct lock_stress_stats *lwsa; /* writer statistics */
  96         struct lock_stress_stats *lrsa; /* reader statistics */
  97 };
  98 static struct lock_torture_cxt cxt = { 0, 0, false,
  99                                        ATOMIC_INIT(0),
 100                                        NULL, NULL};
 101 /*
 102  * Definitions for lock torture testing.
 103  */
 104
 105 static int torture_lock_busted_write_lock(void)
 106 {
 107         return 0;  /* BUGGY, do not use in real life!!! */
 108 }
 109
 110 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
 111 {
 112         const unsigned long longdelay_ms = 100;
 113
 114         /* We want a long delay occasionally to force massive contention.  */
 115         if (!(torture_random(trsp) %
 116               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 117                 mdelay(longdelay_ms);
 118         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 119                 torture_preempt_schedule();  /* Allow test to be preempted. */
 120 }
 121
 122 static void torture_lock_busted_write_unlock(void)
 123 {
 124           /* BUGGY, do not use in real life!!! */
 125 }
 126
 127 static void torture_boost_dummy(struct torture_random_state *trsp)
 128 {
 129         /* Only rtmutexes care about priority */
 130 }
 131
 132 static struct lock_torture_ops lock_busted_ops = {
 133         .writelock      = torture_lock_busted_write_lock,
 134         .write_delay    = torture_lock_busted_write_delay,
 135         .task_boost     = torture_boost_dummy,
 136         .writeunlock    = torture_lock_busted_write_unlock,
 137         .readlock       = NULL,
 138         .read_delay     = NULL,
 139         .readunlock     = NULL,
 140         .name           = "lock_busted"
 141 };
 142
 143 static DEFINE_SPINLOCK(torture_spinlock);
 144
 145 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
 146 {
 147         spin_lock(&torture_spinlock);
 148         return 0;
 149 }
 150
 151 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
 152 {
 153         const unsigned long shortdelay_us = 2;
 154         const unsigned long longdelay_ms = 100;
 155
 156         /* We want a short delay mostly to emulate likely code, and
 157          * we want a long delay occasionally to force massive contention.
 158          */
 159         if (!(torture_random(trsp) %
 160               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 161                 mdelay(longdelay_ms);
 162         if (!(torture_random(trsp) %
 163               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 164                 udelay(shortdelay_us);
 165         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 166                 torture_preempt_schedule();  /* Allow test to be preempted. */
 167 }
 168
 169 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
 170 {
 171         spin_unlock(&torture_spinlock);
 172 }
 173
 174 static struct lock_torture_ops spin_lock_ops = {
 175         .writelock      = torture_spin_lock_write_lock,
 176         .write_delay    = torture_spin_lock_write_delay,
 177         .task_boost     = torture_boost_dummy,
 178         .writeunlock    = torture_spin_lock_write_unlock,
 179         .readlock       = NULL,
 180         .read_delay     = NULL,
 181         .readunlock     = NULL,
 182         .name           = "spin_lock"
 183 };
 184
 185 static int torture_spin_lock_write_lock_irq(void)
 186 __acquires(torture_spinlock)
 187 {
 188         unsigned long flags;
 189
 190         spin_lock_irqsave(&torture_spinlock, flags);
 191         cxt.cur_ops->flags = flags;
 192         return 0;
 193 }
 194
 195 static void torture_lock_spin_write_unlock_irq(void)
 196 __releases(torture_spinlock)
 197 {
 198         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
 199 }
 200
 201 static struct lock_torture_ops spin_lock_irq_ops = {
 202         .writelock      = torture_spin_lock_write_lock_irq,
 203         .write_delay    = torture_spin_lock_write_delay,
 204         .task_boost     = torture_boost_dummy,
 205         .writeunlock    = torture_lock_spin_write_unlock_irq,
 206         .readlock       = NULL,
 207         .read_delay     = NULL,
 208         .readunlock     = NULL,
 209         .name           = "spin_lock_irq"
 210 };
 211
 212 static DEFINE_RWLOCK(torture_rwlock);
 213
 214 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
 215 {
 216         write_lock(&torture_rwlock);
 217         return 0;
 218 }
 219
 220 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
 221 {
 222         const unsigned long shortdelay_us = 2;
 223         const unsigned long longdelay_ms = 100;
 224
 225         /* We want a short delay mostly to emulate likely code, and
 226          * we want a long delay occasionally to force massive contention.
 227          */
 228         if (!(torture_random(trsp) %
 229               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 230                 mdelay(longdelay_ms);
 231         else
 232                 udelay(shortdelay_us);
 233 }
 234
 235 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
 236 {
 237         write_unlock(&torture_rwlock);
 238 }
 239
 240 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
 241 {
 242         read_lock(&torture_rwlock);
 243         return 0;
 244 }
 245
 246 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
 247 {
 248         const unsigned long shortdelay_us = 10;
 249         const unsigned long longdelay_ms = 100;
 250
 251         /* We want a short delay mostly to emulate likely code, and
 252          * we want a long delay occasionally to force massive contention.
 253          */
 254         if (!(torture_random(trsp) %
 255               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 256                 mdelay(longdelay_ms);
 257         else
 258                 udelay(shortdelay_us);
 259 }
 260
 261 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
 262 {
 263         read_unlock(&torture_rwlock);
 264 }
 265
 266 static struct lock_torture_ops rw_lock_ops = {
 267         .writelock      = torture_rwlock_write_lock,
 268         .write_delay    = torture_rwlock_write_delay,
 269         .task_boost     = torture_boost_dummy,
 270         .writeunlock    = torture_rwlock_write_unlock,
 271         .readlock       = torture_rwlock_read_lock,
 272         .read_delay     = torture_rwlock_read_delay,
 273         .readunlock     = torture_rwlock_read_unlock,
 274         .name           = "rw_lock"
 275 };
 276
 277 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
 278 {
 279         unsigned long flags;
 280
 281         write_lock_irqsave(&torture_rwlock, flags);
 282         cxt.cur_ops->flags = flags;
 283         return 0;
 284 }
 285
 286 static void torture_rwlock_write_unlock_irq(void)
 287 __releases(torture_rwlock)
 288 {
 289         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 290 }
 291
 292 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
 293 {
 294         unsigned long flags;
 295
 296         read_lock_irqsave(&torture_rwlock, flags);
 297         cxt.cur_ops->flags = flags;
 298         return 0;
 299 }
 300
 301 static void torture_rwlock_read_unlock_irq(void)
 302 __releases(torture_rwlock)
 303 {
 304         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 305 }
 306
 307 static struct lock_torture_ops rw_lock_irq_ops = {
 308         .writelock      = torture_rwlock_write_lock_irq,
 309         .write_delay    = torture_rwlock_write_delay,
 310         .task_boost     = torture_boost_dummy,
 311         .writeunlock    = torture_rwlock_write_unlock_irq,
 312         .readlock       = torture_rwlock_read_lock_irq,
 313         .read_delay     = torture_rwlock_read_delay,
 314         .readunlock     = torture_rwlock_read_unlock_irq,
 315         .name           = "rw_lock_irq"
 316 };
 317
 318 static DEFINE_MUTEX(torture_mutex);
 319
 320 static int torture_mutex_lock(void) __acquires(torture_mutex)
 321 {
 322         mutex_lock(&torture_mutex);
 323         return 0;
 324 }
 325
 326 static void torture_mutex_delay(struct torture_random_state *trsp)
 327 {
 328         const unsigned long longdelay_ms = 100;
 329
 330         /* We want a long delay occasionally to force massive contention.  */
 331         if (!(torture_random(trsp) %
 332               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 333                 mdelay(longdelay_ms * 5);
 334         else
 335                 mdelay(longdelay_ms / 5);
 336         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 337                 torture_preempt_schedule();  /* Allow test to be preempted. */
 338 }
 339
 340 static void torture_mutex_unlock(void) __releases(torture_mutex)
 341 {
 342         mutex_unlock(&torture_mutex);
 343 }
 344
 345 static struct lock_torture_ops mutex_lock_ops = {
 346         .writelock      = torture_mutex_lock,
 347         .write_delay    = torture_mutex_delay,
 348         .task_boost     = torture_boost_dummy,
 349         .writeunlock    = torture_mutex_unlock,
 350         .readlock       = NULL,
 351         .read_delay     = NULL,
 352         .readunlock     = NULL,
 353         .name           = "mutex_lock"
 354 };
 355
 356 #include <linux/ww_mutex.h>
 357 static DEFINE_WD_CLASS(torture_ww_class);
 358 static DEFINE_WW_MUTEX(torture_ww_mutex_0, &torture_ww_class);
 359 static DEFINE_WW_MUTEX(torture_ww_mutex_1, &torture_ww_class);
 360 static DEFINE_WW_MUTEX(torture_ww_mutex_2, &torture_ww_class);
 361
 362 static int torture_ww_mutex_lock(void)
 363 __acquires(torture_ww_mutex_0)
 364 __acquires(torture_ww_mutex_1)
 365 __acquires(torture_ww_mutex_2)
 366 {
 367         LIST_HEAD(list);
 368         struct reorder_lock {
 369                 struct list_head link;
 370                 struct ww_mutex *lock;
 371         } locks[3], *ll, *ln;
 372         struct ww_acquire_ctx ctx;
 373
 374         locks[0].lock = &torture_ww_mutex_0;
 375         list_add(&locks[0].link, &list);
 376
 377         locks[1].lock = &torture_ww_mutex_1;
 378         list_add(&locks[1].link, &list);
 379
 380         locks[2].lock = &torture_ww_mutex_2;
 381         list_add(&locks[2].link, &list);
 382
 383         ww_acquire_init(&ctx, &torture_ww_class);
 384
 385         list_for_each_entry(ll, &list, link) {
 386                 int err;
 387
 388                 err = ww_mutex_lock(ll->lock, &ctx);
 389                 if (!err)
 390                         continue;
 391
 392                 ln = ll;
 393                 list_for_each_entry_continue_reverse(ln, &list, link)
 394                         ww_mutex_unlock(ln->lock);
 395
 396                 if (err != -EDEADLK)
 397                         return err;
 398
 399                 ww_mutex_lock_slow(ll->lock, &ctx);
 400                 list_move(&ll->link, &list);
 401         }
 402
 403         ww_acquire_fini(&ctx);
 404         return 0;
 405 }
 406
 407 static void torture_ww_mutex_unlock(void)
 408 __releases(torture_ww_mutex_0)
 409 __releases(torture_ww_mutex_1)
 410 __releases(torture_ww_mutex_2)
 411 {
 412         ww_mutex_unlock(&torture_ww_mutex_0);
 413         ww_mutex_unlock(&torture_ww_mutex_1);
 414         ww_mutex_unlock(&torture_ww_mutex_2);
 415 }
 416
 417 static struct lock_torture_ops ww_mutex_lock_ops = {
 418         .writelock      = torture_ww_mutex_lock,
 419         .write_delay    = torture_mutex_delay,
 420         .task_boost     = torture_boost_dummy,
 421         .writeunlock    = torture_ww_mutex_unlock,
 422         .readlock       = NULL,
 423         .read_delay     = NULL,
 424         .readunlock     = NULL,
 425         .name           = "ww_mutex_lock"
 426 };
 427
 428 #ifdef CONFIG_RT_MUTEXES
 429 static DEFINE_RT_MUTEX(torture_rtmutex);
 430
 431 static int torture_rtmutex_lock(void) __acquires(torture_rtmutex)
 432 {
 433         rt_mutex_lock(&torture_rtmutex);
 434         return 0;
 435 }
 436
 437 static void torture_rtmutex_boost(struct torture_random_state *trsp)
 438 {
 439         int policy;
 440         struct sched_param param;
 441         const unsigned int factor = 50000; /* yes, quite arbitrary */
 442
 443         if (!rt_task(current)) {
 444                 /*
 445                  * Boost priority once every ~50k operations. When the
 446                  * task tries to take the lock, the rtmutex it will account
 447                  * for the new priority, and do any corresponding pi-dance.
 448                  */
 449                 if (trsp && !(torture_random(trsp) %
 450                               (cxt.nrealwriters_stress * factor))) {
 451                         policy = SCHED_FIFO;
 452                         param.sched_priority = MAX_RT_PRIO - 1;
 453                 } else /* common case, do nothing */
 454                         return;
 455         } else {
 456                 /*
 457                  * The task will remain boosted for another ~500k operations,
 458                  * then restored back to its original prio, and so forth.
 459                  *
 460                  * When @trsp is nil, we want to force-reset the task for
 461                  * stopping the kthread.
 462                  */
 463                 if (!trsp || !(torture_random(trsp) %
 464                                (cxt.nrealwriters_stress * factor * 2))) {
 465                         policy = SCHED_NORMAL;
 466                         param.sched_priority = 0;
 467                 } else /* common case, do nothing */
 468                         return;
 469         }
 470
 471         sched_setscheduler_nocheck(current, policy, &param);
 472 }
 473
 474 static void torture_rtmutex_delay(struct torture_random_state *trsp)
 475 {
 476         const unsigned long shortdelay_us = 2;
 477         const unsigned long longdelay_ms = 100;
 478
 479         /*
 480          * We want a short delay mostly to emulate likely code, and
 481          * we want a long delay occasionally to force massive contention.
 482          */
 483         if (!(torture_random(trsp) %
 484               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 485                 mdelay(longdelay_ms);
 486         if (!(torture_random(trsp) %
 487               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 488                 udelay(shortdelay_us);
 489         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 490                 torture_preempt_schedule();  /* Allow test to be preempted. */
 491 }
 492
 493 static void torture_rtmutex_unlock(void) __releases(torture_rtmutex)
 494 {
 495         rt_mutex_unlock(&torture_rtmutex);
 496 }
 497
 498 static struct lock_torture_ops rtmutex_lock_ops = {
 499         .writelock      = torture_rtmutex_lock,
 500         .write_delay    = torture_rtmutex_delay,
 501         .task_boost     = torture_rtmutex_boost,
 502         .writeunlock    = torture_rtmutex_unlock,
 503         .readlock       = NULL,
 504         .read_delay     = NULL,
 505         .readunlock     = NULL,
 506         .name           = "rtmutex_lock"
 507 };
 508 #endif
 509
 510 static DECLARE_RWSEM(torture_rwsem);
 511 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
 512 {
 513         down_write(&torture_rwsem);
 514         return 0;
 515 }
 516
 517 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
 518 {
 519         const unsigned long longdelay_ms = 100;
 520
 521         /* We want a long delay occasionally to force massive contention.  */
 522         if (!(torture_random(trsp) %
 523               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 524                 mdelay(longdelay_ms * 10);
 525         else
 526                 mdelay(longdelay_ms / 10);
 527         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 528                 torture_preempt_schedule();  /* Allow test to be preempted. */
 529 }
 530
 531 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
 532 {
 533         up_write(&torture_rwsem);
 534 }
 535
 536 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
 537 {
 538         down_read(&torture_rwsem);
 539         return 0;
 540 }
 541
 542 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
 543 {
 544         const unsigned long longdelay_ms = 100;
 545
 546         /* We want a long delay occasionally to force massive contention.  */
 547         if (!(torture_random(trsp) %
 548               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 549                 mdelay(longdelay_ms * 2);
 550         else
 551                 mdelay(longdelay_ms / 2);
 552         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
 553                 torture_preempt_schedule();  /* Allow test to be preempted. */
 554 }
 555
 556 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
 557 {
 558         up_read(&torture_rwsem);
 559 }
 560
 561 static struct lock_torture_ops rwsem_lock_ops = {
 562         .writelock      = torture_rwsem_down_write,
 563         .write_delay    = torture_rwsem_write_delay,
 564         .task_boost     = torture_boost_dummy,
 565         .writeunlock    = torture_rwsem_up_write,
 566         .readlock       = torture_rwsem_down_read,
 567         .read_delay     = torture_rwsem_read_delay,
 568         .readunlock     = torture_rwsem_up_read,
 569         .name           = "rwsem_lock"
 570 };
 571
 572 #include <linux/percpu-rwsem.h>
 573 static struct percpu_rw_semaphore pcpu_rwsem;
 574
 575 void torture_percpu_rwsem_init(void)
 576 {
 577         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
 578 }
 579
 580 static int torture_percpu_rwsem_down_write(void) __acquires(pcpu_rwsem)
 581 {
 582         percpu_down_write(&pcpu_rwsem);
 583         return 0;
 584 }
 585
 586 static void torture_percpu_rwsem_up_write(void) __releases(pcpu_rwsem)
 587 {
 588         percpu_up_write(&pcpu_rwsem);
 589 }
 590
 591 static int torture_percpu_rwsem_down_read(void) __acquires(pcpu_rwsem)
 592 {
 593         percpu_down_read(&pcpu_rwsem);
 594         return 0;
 595 }
 596
 597 static void torture_percpu_rwsem_up_read(void) __releases(pcpu_rwsem)
 598 {
 599         percpu_up_read(&pcpu_rwsem);
 600 }
 601
 602 static struct lock_torture_ops percpu_rwsem_lock_ops = {
 603         .init           = torture_percpu_rwsem_init,
 604         .writelock      = torture_percpu_rwsem_down_write,
 605         .write_delay    = torture_rwsem_write_delay,
 606         .task_boost     = torture_boost_dummy,
 607         .writeunlock    = torture_percpu_rwsem_up_write,
 608         .readlock       = torture_percpu_rwsem_down_read,
 609         .read_delay     = torture_rwsem_read_delay,
 610         .readunlock     = torture_percpu_rwsem_up_read,
 611         .name           = "percpu_rwsem_lock"
 612 };
 613
 614 /*
 615  * Lock torture writer kthread.  Repeatedly acquires and releases
 616  * the lock, checking for duplicate acquisitions.
 617  */
 618 static int lock_torture_writer(void *arg)
 619 {
 620         struct lock_stress_stats *lwsp = arg;
 621         static DEFINE_TORTURE_RANDOM(rand);
 622
 623         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 624         set_user_nice(current, MAX_NICE);
 625
 626         do {
 627                 if ((torture_random(&rand) & 0xfffff) == 0)
 628                         schedule_timeout_uninterruptible(1);
 629
 630                 cxt.cur_ops->task_boost(&rand);
 631                 cxt.cur_ops->writelock();
 632                 if (WARN_ON_ONCE(lock_is_write_held))
 633                         lwsp->n_lock_fail++;
 634                 lock_is_write_held = 1;
 635                 if (WARN_ON_ONCE(lock_is_read_held))
 636                         lwsp->n_lock_fail++; /* rare, but... */
 637
 638                 lwsp->n_lock_acquired++;
 639                 cxt.cur_ops->write_delay(&rand);
 640                 lock_is_write_held = 0;
 641                 cxt.cur_ops->writeunlock();
 642
 643                 stutter_wait("lock_torture_writer");
 644         } while (!torture_must_stop());
 645
 646         cxt.cur_ops->task_boost(NULL); /* reset prio */
 647         torture_kthread_stopping("lock_torture_writer");
 648         return 0;
 649 }
 650
 651 /*
 652  * Lock torture reader kthread.  Repeatedly acquires and releases
 653  * the reader lock.
 654  */
 655 static int lock_torture_reader(void *arg)
 656 {
 657         struct lock_stress_stats *lrsp = arg;
 658         static DEFINE_TORTURE_RANDOM(rand);
 659
 660         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 661         set_user_nice(current, MAX_NICE);
 662
 663         do {
 664                 if ((torture_random(&rand) & 0xfffff) == 0)
 665                         schedule_timeout_uninterruptible(1);
 666
 667                 cxt.cur_ops->readlock();
 668                 lock_is_read_held = 1;
 669                 if (WARN_ON_ONCE(lock_is_write_held))
 670                         lrsp->n_lock_fail++; /* rare, but... */
 671
 672                 lrsp->n_lock_acquired++;
 673                 cxt.cur_ops->read_delay(&rand);
 674                 lock_is_read_held = 0;
 675                 cxt.cur_ops->readunlock();
 676
 677                 stutter_wait("lock_torture_reader");
 678         } while (!torture_must_stop());
 679         torture_kthread_stopping("lock_torture_reader");
 680         return 0;
 681 }
 682
 683 /*
 684  * Create an lock-torture-statistics message in the specified buffer.
 685  */
 686 static void __torture_print_stats(char *page,
 687                                   struct lock_stress_stats *statp, bool write)
 688 {
 689         bool fail = 0;
 690         int i, n_stress;
 691         long max = 0, min = statp ? statp[0].n_lock_acquired : 0;
 692         long long sum = 0;
 693
 694         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
 695         for (i = 0; i < n_stress; i++) {
 696                 if (statp[i].n_lock_fail)
 697                         fail = true;
 698                 sum += statp[i].n_lock_acquired;
 699                 if (max < statp[i].n_lock_fail)
 700                         max = statp[i].n_lock_fail;
 701                 if (min > statp[i].n_lock_fail)
 702                         min = statp[i].n_lock_fail;
 703         }
 704         page += sprintf(page,
 705                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 706                         write ? "Writes" : "Reads ",
 707                         sum, max, min, max / 2 > min ? "???" : "",
 708                         fail, fail ? "!!!" : "");
 709         if (fail)
 710                 atomic_inc(&cxt.n_lock_torture_errors);
 711 }
 712
 713 /*
 714  * Print torture statistics.  Caller must ensure that there is only one
 715  * call to this function at a given time!!!  This is normally accomplished
 716  * by relying on the module system to only have one copy of the module
 717  * loaded, and then by giving the lock_torture_stats kthread full control
 718  * (or the init/cleanup functions when lock_torture_stats thread is not
 719  * running).
 720  */
 721 static void lock_torture_stats_print(void)
 722 {
 723         int size = cxt.nrealwriters_stress * 200 + 8192;
 724         char *buf;
 725
 726         if (cxt.cur_ops->readlock)
 727                 size += cxt.nrealreaders_stress * 200 + 8192;
 728
 729         buf = kmalloc(size, GFP_KERNEL);
 730         if (!buf) {
 731                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
 732                        size);
 733                 return;
 734         }
 735
 736         __torture_print_stats(buf, cxt.lwsa, true);
 737         pr_alert("%s", buf);
 738         kfree(buf);
 739
 740         if (cxt.cur_ops->readlock) {
 741                 buf = kmalloc(size, GFP_KERNEL);
 742                 if (!buf) {
 743                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
 744                                size);
 745                         return;
 746                 }
 747
 748                 __torture_print_stats(buf, cxt.lrsa, false);
 749                 pr_alert("%s", buf);
 750                 kfree(buf);
 751         }
 752 }
 753
 754 /*
 755  * Periodically prints torture statistics, if periodic statistics printing
 756  * was specified via the stat_interval module parameter.
 757  *
 758  * No need to worry about fullstop here, since this one doesn't reference
 759  * volatile state or register callbacks.
 760  */
 761 static int lock_torture_stats(void *arg)
 762 {
 763         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
 764         do {
 765                 schedule_timeout_interruptible(stat_interval * HZ);
 766                 lock_torture_stats_print();
 767                 torture_shutdown_absorb("lock_torture_stats");
 768         } while (!torture_must_stop());
 769         torture_kthread_stopping("lock_torture_stats");
 770         return 0;
 771 }
 772
 773 static inline void
 774 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
 775                                 const char *tag)
 776 {
 777         pr_alert("%s" TORTURE_FLAG
 778                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
 779                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
 780                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
 781                  verbose, shuffle_interval, stutter, shutdown_secs,
 782                  onoff_interval, onoff_holdoff);
 783 }
 784
 785 static void lock_torture_cleanup(void)
 786 {
 787         int i;
 788
 789         if (torture_cleanup_begin())
 790                 return;
 791
 792         /*
 793          * Indicates early cleanup, meaning that the test has not run,
 794          * such as when passing bogus args when loading the module. As
 795          * such, only perform the underlying torture-specific cleanups,
 796          * and avoid anything related to locktorture.
 797          */
 798         if (!cxt.lwsa && !cxt.lrsa)
 799                 goto end;
 800
 801         if (writer_tasks) {
 802                 for (i = 0; i < cxt.nrealwriters_stress; i++)
 803                         torture_stop_kthread(lock_torture_writer,
 804                                              writer_tasks[i]);
 805                 kfree(writer_tasks);
 806                 writer_tasks = NULL;
 807         }
 808
 809         if (reader_tasks) {
 810                 for (i = 0; i < cxt.nrealreaders_stress; i++)
 811                         torture_stop_kthread(lock_torture_reader,
 812                                              reader_tasks[i]);
 813                 kfree(reader_tasks);
 814                 reader_tasks = NULL;
 815         }
 816
 817         torture_stop_kthread(lock_torture_stats, stats_task);
 818         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
 819
 820         if (atomic_read(&cxt.n_lock_torture_errors))
 821                 lock_torture_print_module_parms(cxt.cur_ops,
 822                                                 "End of test: FAILURE");
 823         else if (torture_onoff_failures())
 824                 lock_torture_print_module_parms(cxt.cur_ops,
 825                                                 "End of test: LOCK_HOTPLUG");
 826         else
 827                 lock_torture_print_module_parms(cxt.cur_ops,
 828                                                 "End of test: SUCCESS");
 829
 830         kfree(cxt.lwsa);
 831         cxt.lwsa = NULL;
 832         kfree(cxt.lrsa);
 833         cxt.lrsa = NULL;
 834
 835 end:
 836         torture_cleanup_end();
 837 }
 838
 839 static int __init lock_torture_init(void)
 840 {
 841         int i, j;
 842         int firsterr = 0;
 843         static struct lock_torture_ops *torture_ops[] = {
 844                 &lock_busted_ops,
 845                 &spin_lock_ops, &spin_lock_irq_ops,
 846                 &rw_lock_ops, &rw_lock_irq_ops,
 847                 &mutex_lock_ops,
 848                 &ww_mutex_lock_ops,
 849 #ifdef CONFIG_RT_MUTEXES
 850                 &rtmutex_lock_ops,
 851 #endif
 852                 &rwsem_lock_ops,
 853                 &percpu_rwsem_lock_ops,
 854         };
 855
 856         if (!torture_init_begin(torture_type, verbose))
 857                 return -EBUSY;
 858
 859         /* Process args and tell the world that the torturer is on the job. */
 860         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
 861                 cxt.cur_ops = torture_ops[i];
 862                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
 863                         break;
 864         }
 865         if (i == ARRAY_SIZE(torture_ops)) {
 866                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
 867                          torture_type);
 868                 pr_alert("lock-torture types:");
 869                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
 870                         pr_alert(" %s", torture_ops[i]->name);
 871                 pr_alert("\n");
 872                 firsterr = -EINVAL;
 873                 goto unwind;
 874         }
 875
 876         if (nwriters_stress == 0 && nreaders_stress == 0) {
 877                 pr_alert("lock-torture: must run at least one locking thread\n");
 878                 firsterr = -EINVAL;
 879                 goto unwind;
 880         }
 881
 882         if (cxt.cur_ops->init)
 883                 cxt.cur_ops->init();
 884
 885         if (nwriters_stress >= 0)
 886                 cxt.nrealwriters_stress = nwriters_stress;
 887         else
 888                 cxt.nrealwriters_stress = 2 * num_online_cpus();
 889
 890 #ifdef CONFIG_DEBUG_MUTEXES
 891         if (str_has_prefix(torture_type, "mutex"))
 892                 cxt.debug_lock = true;
 893 #endif
 894 #ifdef CONFIG_DEBUG_RT_MUTEXES
 895         if (str_has_prefix(torture_type, "rtmutex"))
 896                 cxt.debug_lock = true;
 897 #endif
 898 #ifdef CONFIG_DEBUG_SPINLOCK
 899         if ((str_has_prefix(torture_type, "spin")) ||
 900             (str_has_prefix(torture_type, "rw_lock")))
 901                 cxt.debug_lock = true;
 902 #endif
 903
 904         /* Initialize the statistics so that each run gets its own numbers. */
 905         if (nwriters_stress) {
 906                 lock_is_write_held = 0;
 907                 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
 908                                          sizeof(*cxt.lwsa),
 909                                          GFP_KERNEL);
 910                 if (cxt.lwsa == NULL) {
 911                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
 912                         firsterr = -ENOMEM;
 913                         goto unwind;
 914                 }
 915
 916                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
 917                         cxt.lwsa[i].n_lock_fail = 0;
 918                         cxt.lwsa[i].n_lock_acquired = 0;
 919                 }
 920         }
 921
 922         if (cxt.cur_ops->readlock) {
 923                 if (nreaders_stress >= 0)
 924                         cxt.nrealreaders_stress = nreaders_stress;
 925                 else {
 926                         /*
 927                          * By default distribute evenly the number of
 928                          * readers and writers. We still run the same number
 929                          * of threads as the writer-only locks default.
 930                          */
 931                         if (nwriters_stress < 0) /* user doesn't care */
 932                                 cxt.nrealwriters_stress = num_online_cpus();
 933                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
 934                 }
 935
 936                 if (nreaders_stress) {
 937                         lock_is_read_held = 0;
 938                         cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
 939                                                  sizeof(*cxt.lrsa),
 940                                                  GFP_KERNEL);
 941                         if (cxt.lrsa == NULL) {
 942                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
 943                                 firsterr = -ENOMEM;
 944                                 kfree(cxt.lwsa);
 945                                 cxt.lwsa = NULL;
 946                                 goto unwind;
 947                         }
 948
 949                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
 950                                 cxt.lrsa[i].n_lock_fail = 0;
 951                                 cxt.lrsa[i].n_lock_acquired = 0;
 952                         }
 953                 }
 954         }
 955
 956         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
 957
 958         /* Prepare torture context. */
 959         if (onoff_interval > 0) {
 960                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
 961                                               onoff_interval * HZ, NULL);
 962                 if (firsterr)
 963                         goto unwind;
 964         }
 965         if (shuffle_interval > 0) {
 966                 firsterr = torture_shuffle_init(shuffle_interval);
 967                 if (firsterr)
 968                         goto unwind;
 969         }
 970         if (shutdown_secs > 0) {
 971                 firsterr = torture_shutdown_init(shutdown_secs,
 972                                                  lock_torture_cleanup);
 973                 if (firsterr)
 974                         goto unwind;
 975         }
 976         if (stutter > 0) {
 977                 firsterr = torture_stutter_init(stutter, stutter);
 978                 if (firsterr)
 979                         goto unwind;
 980         }
 981
 982         if (nwriters_stress) {
 983                 writer_tasks = kcalloc(cxt.nrealwriters_stress,
 984                                        sizeof(writer_tasks[0]),
 985                                        GFP_KERNEL);
 986                 if (writer_tasks == NULL) {
 987                         VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
 988                         firsterr = -ENOMEM;
 989                         goto unwind;
 990                 }
 991         }
 992
 993         if (cxt.cur_ops->readlock) {
 994                 reader_tasks = kcalloc(cxt.nrealreaders_stress,
 995                                        sizeof(reader_tasks[0]),
 996                                        GFP_KERNEL);
 997                 if (reader_tasks == NULL) {
 998                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
 999                         kfree(writer_tasks);
1000                         writer_tasks = NULL;
1001                         firsterr = -ENOMEM;
1002                         goto unwind;
1003                 }
1004         }
1005
1006         /*
1007          * Create the kthreads and start torturing (oh, those poor little locks).
1008          *
1009          * TODO: Note that we interleave writers with readers, giving writers a
1010          * slight advantage, by creating its kthread first. This can be modified
1011          * for very specific needs, or even let the user choose the policy, if
1012          * ever wanted.
1013          */
1014         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1015                     j < cxt.nrealreaders_stress; i++, j++) {
1016                 if (i >= cxt.nrealwriters_stress)
1017                         goto create_reader;
1018
1019                 /* Create writer. */
1020                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
1021                                                   writer_tasks[i]);
1022                 if (firsterr)
1023                         goto unwind;
1024
1025         create_reader:
1026                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1027                         continue;
1028                 /* Create reader. */
1029                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1030                                                   reader_tasks[j]);
1031                 if (firsterr)
1032                         goto unwind;
1033         }
1034         if (stat_interval > 0) {
1035                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1036                                                   stats_task);
1037                 if (firsterr)
1038                         goto unwind;
1039         }
1040         torture_init_end();
1041         return 0;
1042
1043 unwind:
1044         torture_init_end();
1045         lock_torture_cleanup();
1046         return firsterr;
1047 }
1048
1049 module_init(lock_torture_init);
1050 module_exit(lock_torture_cleanup);