rcutorture: Test polled expedited grace-period primitives

This commit adds tests of start_poll_synchronize_rcu_expedited() and
poll_state_synchronize_rcu_expedited().

Link: https://lore.kernel.org/all/20220121142454.1994916-1-bfoster@redhat.com/
Link: https://docs.google.com/document/d/1RNKWW9jQyfjxw2E8dsXVTdvZYh0HnYeSHDKog9jhdN8/edit?usp=sharing
Cc: Brian Foster <bfoster@redhat.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Ian Kent <raven@themaw.net>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index d2edc76..0788ef2 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -86,10 +86,12 @@ torture_param(int, fwd_progress_holdoff, 60,
 torture_param(bool, fwd_progress_need_resched, 1,
              "Hide cond_resched() behind need_resched()");
 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
+torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives");
 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
 torture_param(bool, gp_normal, false,
             "Use normal (non-expedited) GP wait primitives");
 torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
+torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives");
 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
@@ -209,12 +211,16 @@ static int rcu_torture_writer_state;
 #define RTWS_DEF_FREE          3
 #define RTWS_EXP_SYNC          4
 #define RTWS_COND_GET          5
-#define RTWS_COND_SYNC         6
-#define RTWS_POLL_GET          7
-#define RTWS_POLL_WAIT         8
-#define RTWS_SYNC              9
-#define RTWS_STUTTER           10
-#define RTWS_STOPPING          11
+#define RTWS_COND_GET_EXP      6
+#define RTWS_COND_SYNC         7
+#define RTWS_COND_SYNC_EXP     8
+#define RTWS_POLL_GET          9
+#define RTWS_POLL_GET_EXP      10
+#define RTWS_POLL_WAIT         11
+#define RTWS_POLL_WAIT_EXP     12
+#define RTWS_SYNC              13
+#define RTWS_STUTTER           14
+#define RTWS_STOPPING          15
 static const char * const rcu_torture_writer_state_names[] = {
        "RTWS_FIXED_DELAY",
        "RTWS_DELAY",
@@ -222,9 +228,13 @@ static const char * const rcu_torture_writer_state_names[] = {
        "RTWS_DEF_FREE",
        "RTWS_EXP_SYNC",
        "RTWS_COND_GET",
+       "RTWS_COND_GET_EXP",
        "RTWS_COND_SYNC",
+       "RTWS_COND_SYNC_EXP",
        "RTWS_POLL_GET",
+       "RTWS_POLL_GET_EXP",
        "RTWS_POLL_WAIT",
+       "RTWS_POLL_WAIT_EXP",
        "RTWS_SYNC",
        "RTWS_STUTTER",
        "RTWS_STOPPING",
@@ -337,6 +347,10 @@ struct rcu_torture_ops {
        void (*deferred_free)(struct rcu_torture *p);
        void (*sync)(void);
        void (*exp_sync)(void);
+       unsigned long (*get_gp_state_exp)(void);
+       unsigned long (*start_gp_poll_exp)(void);
+       bool (*poll_gp_state_exp)(unsigned long oldstate);
+       void (*cond_sync_exp)(unsigned long oldstate);
        unsigned long (*get_gp_state)(void);
        unsigned long (*get_gp_completed)(void);
        unsigned long (*start_gp_poll)(void);
@@ -509,6 +523,10 @@ static struct rcu_torture_ops rcu_ops = {
        .start_gp_poll          = start_poll_synchronize_rcu,
        .poll_gp_state          = poll_state_synchronize_rcu,
        .cond_sync              = cond_synchronize_rcu,
+       .get_gp_state_exp       = get_state_synchronize_rcu,
+       .start_gp_poll_exp      = start_poll_synchronize_rcu_expedited,
+       .poll_gp_state_exp      = poll_state_synchronize_rcu,
+       .cond_sync_exp          = cond_synchronize_rcu_expedited,
        .call                   = call_rcu,
        .cb_barrier             = rcu_barrier,
        .fqs                    = rcu_force_quiescent_state,
@@ -1138,9 +1156,8 @@ rcu_torture_fqs(void *arg)
        return 0;
 }
 
-// Used by writers to randomly choose from the available grace-period
-// primitives.  The only purpose of the initialization is to size the array.
-static int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, RTWS_COND_GET, RTWS_POLL_GET, RTWS_SYNC };
+// Used by writers to randomly choose from the available grace-period primitives.
+static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { };
 static int nsynctypes;
 
 /*
@@ -1148,18 +1165,27 @@ static int nsynctypes;
  */
 static void rcu_torture_write_types(void)
 {
-       bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
-       bool gp_poll1 = gp_poll, gp_sync1 = gp_sync;
+       bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp;
+       bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll;
+       bool gp_sync1 = gp_sync;
 
        /* Initialize synctype[] array.  If none set, take default. */
-       if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_poll1 && !gp_sync1)
-               gp_cond1 = gp_exp1 = gp_normal1 = gp_poll1 = gp_sync1 = true;
+       if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp &&
+           !gp_normal1 && !gp_poll1 && !gp_sync1)
+               gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 =
+                          gp_normal1 = gp_poll1 = gp_sync1 = true;
        if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
                synctype[nsynctypes++] = RTWS_COND_GET;
                pr_info("%s: Testing conditional GPs.\n", __func__);
        } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {
                pr_alert("%s: gp_cond without primitives.\n", __func__);
        }
+       if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) {
+               synctype[nsynctypes++] = RTWS_COND_GET_EXP;
+               pr_info("%s: Testing conditional expedited GPs.\n", __func__);
+       } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {
+               pr_alert("%s: gp_cond_exp without primitives.\n", __func__);
+       }
        if (gp_exp1 && cur_ops->exp_sync) {
                synctype[nsynctypes++] = RTWS_EXP_SYNC;
                pr_info("%s: Testing expedited GPs.\n", __func__);
@@ -1178,6 +1204,12 @@ static void rcu_torture_write_types(void)
        } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
                pr_alert("%s: gp_poll without primitives.\n", __func__);
        }
+       if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {
+               synctype[nsynctypes++] = RTWS_POLL_GET_EXP;
+               pr_info("%s: Testing polling expedited GPs.\n", __func__);
+       } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {
+               pr_alert("%s: gp_poll_exp without primitives.\n", __func__);
+       }
        if (gp_sync1 && cur_ops->sync) {
                synctype[nsynctypes++] = RTWS_SYNC;
                pr_info("%s: Testing normal GPs.\n", __func__);
@@ -1285,6 +1317,14 @@ rcu_torture_writer(void *arg)
                                cur_ops->cond_sync(gp_snap);
                                rcu_torture_pipe_update(old_rp);
                                break;
+                       case RTWS_COND_GET_EXP:
+                               rcu_torture_writer_state = RTWS_COND_GET_EXP;
+                               gp_snap = cur_ops->get_gp_state_exp();
+                               torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+                               rcu_torture_writer_state = RTWS_COND_SYNC_EXP;
+                               cur_ops->cond_sync_exp(gp_snap);
+                               rcu_torture_pipe_update(old_rp);
+                               break;
                        case RTWS_POLL_GET:
                                rcu_torture_writer_state = RTWS_POLL_GET;
                                gp_snap = cur_ops->start_gp_poll();
@@ -1294,6 +1334,15 @@ rcu_torture_writer(void *arg)
                                                                  &rand);
                                rcu_torture_pipe_update(old_rp);
                                break;
+                       case RTWS_POLL_GET_EXP:
+                               rcu_torture_writer_state = RTWS_POLL_GET_EXP;
+                               gp_snap = cur_ops->start_gp_poll_exp();
+                               rcu_torture_writer_state = RTWS_POLL_WAIT_EXP;
+                               while (!cur_ops->poll_gp_state_exp(gp_snap))
+                                       torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+                                                                 &rand);
+                               rcu_torture_pipe_update(old_rp);
+                               break;
                        case RTWS_SYNC:
                                rcu_torture_writer_state = RTWS_SYNC;
                                if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
@@ -1400,6 +1449,11 @@ rcu_torture_fakewriter(void *arg)
                                torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
                                cur_ops->cond_sync(gp_snap);
                                break;
+                       case RTWS_COND_GET_EXP:
+                               gp_snap = cur_ops->get_gp_state_exp();
+                               torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+                               cur_ops->cond_sync_exp(gp_snap);
+                               break;
                        case RTWS_POLL_GET:
                                gp_snap = cur_ops->start_gp_poll();
                                while (!cur_ops->poll_gp_state(gp_snap)) {
@@ -1407,6 +1461,13 @@ rcu_torture_fakewriter(void *arg)
                                                                  &rand);
                                }
                                break;
+                       case RTWS_POLL_GET_EXP:
+                               gp_snap = cur_ops->start_gp_poll_exp();
+                               while (!cur_ops->poll_gp_state_exp(gp_snap)) {
+                                       torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+                                                                 &rand);
+                               }
+                               break;
                        case RTWS_SYNC:
                                cur_ops->sync();
                                break;