rcu: Introduce rcutorture testing for rcu_barrier()

Although rcutorture does invoke rcu_barrier() and friends, it cannot
really be called a torture test given that it invokes them only once
at the end of the test.  This commit therefore introduces heavy-duty
rcutorture testing for rcu_barrier(), which may be carried out
concurrently with normal rcutorture testing.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/Documentation/RCU/torture.txt b/Documentation/RCU/torture.txt
index 375d3fb..4ddf391 100644 (file)
--- a/Documentation/RCU/torture.txt
+++ b/Documentation/RCU/torture.txt
@@ -47,6 +47,16 @@ irqreader    Says to invoke RCU readers from irq level.  This is currently
                permit this.  (Or, more accurately, variants of RCU that do
                -not- permit this know to ignore this variable.)
 
+n_barrier_cbs  If this is nonzero, RCU barrier testing will be conducted,
+               in which case n_barrier_cbs specifies the number of
+               RCU callbacks (and corresponding kthreads) to use for
+               this testing.  The value cannot be negative.  If you
+               specify this to be non-zero when torture_type indicates a
+               synchronous RCU implementation (one for which a member of
+               the synchronize_rcu() rather than the call_rcu() family is
+               used -- see the documentation for torture_type below), an
+               error will be reported and no testing will be carried out.
+
 nfakewriters   This is the number of RCU fake writer threads to run.  Fake
                writer threads repeatedly use the synchronous "wait for
                current readers" function of the interface selected by
@@ -188,7 +198,7 @@ OUTPUT
 The statistics output is as follows:
 
        rcu-torture:--- Start of test: nreaders=16 nfakewriters=4 stat_interval=30 verbose=0 test_no_idle_hz=1 shuffle_interval=3 stutter=5 irqreader=1 fqs_duration=0 fqs_holdoff=0 fqs_stutter=3 test_boost=1/0 test_boost_interval=7 test_boost_duration=4
-       rcu-torture: rtc:           (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767
+       rcu-torture: rtc:           (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767
        rcu-torture: Reader Pipe:  727860534 34213 0 0 0 0 0 0 0 0 0
        rcu-torture: Reader Batch:  727877838 17003 0 0 0 0 0 0 0 0 0
        rcu-torture: Free-Block Circulation:  155440 155440 155440 155440 155440 155440 155440 155440 155440 155440 0
@@ -230,6 +240,9 @@ o   "rtmbe": A non-zero value indicates that rcutorture believes that
        rcu_assign_pointer() and rcu_dereference() are not working
        correctly.  This value should be zero.
 
+o      "rtbe": A non-zero value indicates that one of the rcu_barrier()
+       family of functions is not working correctly.
+
 o      "rtbke": rcutorture was unable to create the real-time kthreads
        used to force RCU priority inversion.  This value should be zero.
 

diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 1463a06..8cd262b 100644 (file)
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -64,6 +64,7 @@ static int irqreader = 1;     /* RCU readers from irq (timers). */
 static int fqs_duration;       /* Duration of bursts (us), 0 to disable. */
 static int fqs_holdoff;                /* Hold time within burst (us). */
 static int fqs_stutter = 3;    /* Wait time between bursts (s). */
+static int n_barrier_cbs;      /* Number of callbacks to test RCU barriers. */
 static int onoff_interval;     /* Wait time between CPU hotplugs, 0=disable. */
 static int onoff_holdoff;      /* Seconds after boot before CPU hotplugs. */
 static int shutdown_secs;      /* Shutdown time (s).  <=0 for no shutdown. */
@@ -96,6 +97,8 @@ module_param(fqs_holdoff, int, 0444);
 MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
 module_param(fqs_stutter, int, 0444);
 MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
+module_param(n_barrier_cbs, int, 0444);
+MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
 module_param(onoff_interval, int, 0444);
 MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
 module_param(onoff_holdoff, int, 0444);
@@ -139,6 +142,8 @@ static struct task_struct *shutdown_task;
 static struct task_struct *onoff_task;
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 static struct task_struct *stall_task;
+static struct task_struct **barrier_cbs_tasks;
+static struct task_struct *barrier_task;
 
 #define RCU_TORTURE_PIPE_LEN 10
 
@@ -164,6 +169,7 @@ static atomic_t n_rcu_torture_alloc_fail;
 static atomic_t n_rcu_torture_free;
 static atomic_t n_rcu_torture_mberror;
 static atomic_t n_rcu_torture_error;
+static long n_rcu_torture_barrier_error;
 static long n_rcu_torture_boost_ktrerror;
 static long n_rcu_torture_boost_rterror;
 static long n_rcu_torture_boost_failure;
@@ -173,6 +179,8 @@ static long n_offline_attempts;
 static long n_offline_successes;
 static long n_online_attempts;
 static long n_online_successes;
+static long n_barrier_attempts;
+static long n_barrier_successes;
 static struct list_head rcu_torture_removed;
 static cpumask_var_t shuffle_tmp_mask;
 
@@ -197,6 +205,10 @@ static unsigned long shutdown_time;        /* jiffies to system shutdown. */
 static unsigned long boost_starttime;  /* jiffies of next boost test start. */
 DEFINE_MUTEX(boost_mutex);             /* protect setting boost_starttime */
                                        /*  and boost task create/destroy. */
+static atomic_t barrier_cbs_count;     /* Barrier callbacks registered. */
+static atomic_t barrier_cbs_invoked;   /* Barrier callbacks invoked. */
+static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
+static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
 
 /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
 
@@ -327,6 +339,7 @@ struct rcu_torture_ops {
        int (*completed)(void);
        void (*deferred_free)(struct rcu_torture *p);
        void (*sync)(void);
+       void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
        void (*cb_barrier)(void);
        void (*fqs)(void);
        int (*stats)(char *page);
@@ -417,6 +430,7 @@ static struct rcu_torture_ops rcu_ops = {
        .completed      = rcu_torture_completed,
        .deferred_free  = rcu_torture_deferred_free,
        .sync           = synchronize_rcu,
+       .call           = call_rcu,
        .cb_barrier     = rcu_barrier,
        .fqs            = rcu_force_quiescent_state,
        .stats          = NULL,
@@ -460,6 +474,7 @@ static struct rcu_torture_ops rcu_sync_ops = {
        .completed      = rcu_torture_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = synchronize_rcu,
+       .call           = NULL,
        .cb_barrier     = NULL,
        .fqs            = rcu_force_quiescent_state,
        .stats          = NULL,
@@ -477,6 +492,7 @@ static struct rcu_torture_ops rcu_expedited_ops = {
        .completed      = rcu_no_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = synchronize_rcu_expedited,
+       .call           = NULL,
        .cb_barrier     = NULL,
        .fqs            = rcu_force_quiescent_state,
        .stats          = NULL,
@@ -519,6 +535,7 @@ static struct rcu_torture_ops rcu_bh_ops = {
        .completed      = rcu_bh_torture_completed,
        .deferred_free  = rcu_bh_torture_deferred_free,
        .sync           = synchronize_rcu_bh,
+       .call           = call_rcu_bh,
        .cb_barrier     = rcu_barrier_bh,
        .fqs            = rcu_bh_force_quiescent_state,
        .stats          = NULL,
@@ -535,6 +552,7 @@ static struct rcu_torture_ops rcu_bh_sync_ops = {
        .completed      = rcu_bh_torture_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = synchronize_rcu_bh,
+       .call           = NULL,
        .cb_barrier     = NULL,
        .fqs            = rcu_bh_force_quiescent_state,
        .stats          = NULL,
@@ -551,6 +569,7 @@ static struct rcu_torture_ops rcu_bh_expedited_ops = {
        .completed      = rcu_bh_torture_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = synchronize_rcu_bh_expedited,
+       .call           = NULL,
        .cb_barrier     = NULL,
        .fqs            = rcu_bh_force_quiescent_state,
        .stats          = NULL,
@@ -637,6 +656,7 @@ static struct rcu_torture_ops srcu_ops = {
        .completed      = srcu_torture_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = srcu_torture_synchronize,
+       .call           = NULL,
        .cb_barrier     = NULL,
        .stats          = srcu_torture_stats,
        .name           = "srcu"
@@ -661,6 +681,7 @@ static struct rcu_torture_ops srcu_raw_ops = {
        .completed      = srcu_torture_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = srcu_torture_synchronize,
+       .call           = NULL,
        .cb_barrier     = NULL,
        .stats          = srcu_torture_stats,
        .name           = "srcu_raw"
@@ -680,6 +701,7 @@ static struct rcu_torture_ops srcu_expedited_ops = {
        .completed      = srcu_torture_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = srcu_torture_synchronize_expedited,
+       .call           = NULL,
        .cb_barrier     = NULL,
        .stats          = srcu_torture_stats,
        .name           = "srcu_expedited"
@@ -1129,7 +1151,8 @@ rcu_torture_printk(char *page)
                       "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d "
                       "rtmbe: %d rtbke: %ld rtbre: %ld "
                       "rtbf: %ld rtb: %ld nt: %ld "
-                      "onoff: %ld/%ld:%ld/%ld",
+                      "onoff: %ld/%ld:%ld/%ld "
+                      "barrier: %ld/%ld:%ld",
                       rcu_torture_current,
                       rcu_torture_current_version,
                       list_empty(&rcu_torture_freelist),
@@ -1145,14 +1168,17 @@ rcu_torture_printk(char *page)
                       n_online_successes,
                       n_online_attempts,
                       n_offline_successes,
-                      n_offline_attempts);
+                      n_offline_attempts,
+                      n_barrier_successes,
+                      n_barrier_attempts,
+                      n_rcu_torture_barrier_error);
+       cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
        if (atomic_read(&n_rcu_torture_mberror) != 0 ||
+           n_rcu_torture_barrier_error != 0 ||
            n_rcu_torture_boost_ktrerror != 0 ||
            n_rcu_torture_boost_rterror != 0 ||
-           n_rcu_torture_boost_failure != 0)
-               cnt += sprintf(&page[cnt], " !!!");
-       cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
-       if (i > 1) {
+           n_rcu_torture_boost_failure != 0 ||
+           i > 1) {
                cnt += sprintf(&page[cnt], "!!! ");
                atomic_inc(&n_rcu_torture_error);
                WARN_ON_ONCE(1);
@@ -1560,6 +1586,151 @@ static void rcu_torture_stall_cleanup(void)
        stall_task = NULL;
 }
 
+/* Callback function for RCU barrier testing. */
+void rcu_torture_barrier_cbf(struct rcu_head *rcu)
+{
+       atomic_inc(&barrier_cbs_invoked);
+}
+
+/* kthread function to register callbacks used to test RCU barriers. */
+static int rcu_torture_barrier_cbs(void *arg)
+{
+       long myid = (long)arg;
+       struct rcu_head rcu;
+
+       init_rcu_head_on_stack(&rcu);
+       VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task started");
+       set_user_nice(current, 19);
+       do {
+               wait_event(barrier_cbs_wq[myid],
+                          atomic_read(&barrier_cbs_count) == n_barrier_cbs ||
+                          kthread_should_stop() ||
+                          fullstop != FULLSTOP_DONTSTOP);
+               if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
+                       break;
+               cur_ops->call(&rcu, rcu_torture_barrier_cbf);
+               if (atomic_dec_and_test(&barrier_cbs_count))
+                       wake_up(&barrier_wq);
+       } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
+       VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task stopping");
+       rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
+       while (!kthread_should_stop())
+               schedule_timeout_interruptible(1);
+       cur_ops->cb_barrier();
+       destroy_rcu_head_on_stack(&rcu);
+       return 0;
+}
+
+/* kthread function to drive and coordinate RCU barrier testing. */
+static int rcu_torture_barrier(void *arg)
+{
+       int i;
+
+       VERBOSE_PRINTK_STRING("rcu_torture_barrier task starting");
+       do {
+               atomic_set(&barrier_cbs_invoked, 0);
+               atomic_set(&barrier_cbs_count, n_barrier_cbs);
+               /* wake_up() path contains the required barriers. */
+               for (i = 0; i < n_barrier_cbs; i++)
+                       wake_up(&barrier_cbs_wq[i]);
+               wait_event(barrier_wq,
+                          atomic_read(&barrier_cbs_count) == 0 ||
+                          kthread_should_stop() ||
+                          fullstop != FULLSTOP_DONTSTOP);
+               if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
+                       break;
+               n_barrier_attempts++;
+               cur_ops->cb_barrier();
+               if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
+                       n_rcu_torture_barrier_error++;
+                       WARN_ON_ONCE(1);
+               }
+               n_barrier_successes++;
+               schedule_timeout_interruptible(HZ / 10);
+       } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
+       VERBOSE_PRINTK_STRING("rcu_torture_barrier task stopping");
+       rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
+       while (!kthread_should_stop())
+               schedule_timeout_interruptible(1);
+       return 0;
+}
+
+/* Initialize RCU barrier testing. */
+static int rcu_torture_barrier_init(void)
+{
+       int i;
+       int ret;
+
+       if (n_barrier_cbs == 0)
+               return 0;
+       if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
+               printk(KERN_ALERT "%s" TORTURE_FLAG
+                      " Call or barrier ops missing for %s,\n",
+                      torture_type, cur_ops->name);
+               printk(KERN_ALERT "%s" TORTURE_FLAG
+                      " RCU barrier testing omitted from run.\n",
+                      torture_type);
+               return 0;
+       }
+       atomic_set(&barrier_cbs_count, 0);
+       atomic_set(&barrier_cbs_invoked, 0);
+       barrier_cbs_tasks =
+               kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
+                       GFP_KERNEL);
+       barrier_cbs_wq =
+               kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
+                       GFP_KERNEL);
+       if (barrier_cbs_tasks == NULL || barrier_cbs_wq == 0)
+               return -ENOMEM;
+       for (i = 0; i < n_barrier_cbs; i++) {
+               init_waitqueue_head(&barrier_cbs_wq[i]);
+               barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs,
+                                                  (void *)i,
+                                                  "rcu_torture_barrier_cbs");
+               if (IS_ERR(barrier_cbs_tasks[i])) {
+                       ret = PTR_ERR(barrier_cbs_tasks[i]);
+                       VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier_cbs");
+                       barrier_cbs_tasks[i] = NULL;
+                       return ret;
+               }
+       }
+       barrier_task = kthread_run(rcu_torture_barrier, NULL,
+                                  "rcu_torture_barrier");
+       if (IS_ERR(barrier_task)) {
+               ret = PTR_ERR(barrier_task);
+               VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier");
+               barrier_task = NULL;
+       }
+       return 0;
+}
+
+/* Clean up after RCU barrier testing. */
+static void rcu_torture_barrier_cleanup(void)
+{
+       int i;
+
+       if (barrier_task != NULL) {
+               VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier task");
+               kthread_stop(barrier_task);
+               barrier_task = NULL;
+       }
+       if (barrier_cbs_tasks != NULL) {
+               for (i = 0; i < n_barrier_cbs; i++) {
+                       if (barrier_cbs_tasks[i] != NULL) {
+                               VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier_cbs task");
+                               kthread_stop(barrier_cbs_tasks[i]);
+                               barrier_cbs_tasks[i] = NULL;
+                       }
+               }
+               kfree(barrier_cbs_tasks);
+               barrier_cbs_tasks = NULL;
+       }
+       if (barrier_cbs_wq != NULL) {
+               kfree(barrier_cbs_wq);
+               barrier_cbs_wq = NULL;
+       }
+}
+
 static int rcutorture_cpu_notify(struct notifier_block *self,
                                 unsigned long action, void *hcpu)
 {
@@ -1602,6 +1773,7 @@ rcu_torture_cleanup(void)
        fullstop = FULLSTOP_RMMOD;
        mutex_unlock(&fullstop_mutex);
        unregister_reboot_notifier(&rcutorture_shutdown_nb);
+       rcu_torture_barrier_cleanup();
        rcu_torture_stall_cleanup();
        if (stutter_task) {
                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
@@ -1681,7 +1853,7 @@ rcu_torture_cleanup(void)
 
        if (cur_ops->cleanup)
                cur_ops->cleanup();
-       if (atomic_read(&n_rcu_torture_error))
+       if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
                rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
        else if (n_online_successes != n_online_attempts ||
                 n_offline_successes != n_offline_attempts)
@@ -1697,6 +1869,7 @@ rcu_torture_init(void)
        int i;
        int cpu;
        int firsterr = 0;
+       int retval;
        static struct rcu_torture_ops *torture_ops[] =
                { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
                  &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
@@ -1754,6 +1927,7 @@ rcu_torture_init(void)
        atomic_set(&n_rcu_torture_free, 0);
        atomic_set(&n_rcu_torture_mberror, 0);
        atomic_set(&n_rcu_torture_error, 0);
+       n_rcu_torture_barrier_error = 0;
        n_rcu_torture_boost_ktrerror = 0;
        n_rcu_torture_boost_rterror = 0;
        n_rcu_torture_boost_failure = 0;
@@ -1877,7 +2051,6 @@ rcu_torture_init(void)
                test_boost_duration = 2;
        if ((test_boost == 1 && cur_ops->can_boost) ||
            test_boost == 2) {
-               int retval;
 
                boost_starttime = jiffies + test_boost_interval * HZ;
                register_cpu_notifier(&rcutorture_cpu_nb);
@@ -1913,6 +2086,11 @@ rcu_torture_init(void)
                firsterr = i;
                goto unwind;
        }
+       retval = rcu_torture_barrier_init();
+       if (retval != 0) {
+               firsterr = retval;
+               goto unwind;
+       }
        rcutorture_record_test_transition();
        mutex_unlock(&fullstop_mutex);
        return 0;