srcu: Automatically determine size-transition strategy at boot

This commit adds a srcutree.convert_to_big option of zero that causes
SRCU to decide at boot whether to wait for contention (small systems) or
immediately expand to large (large systems).  A new srcutree.big_cpu_lim
(defaulting to 128) defines how many CPUs constitute a large system.

Co-developed-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 177e688..0a094bb 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5608,6 +5608,15 @@
                        off:    Disable mitigation and remove
                                performance impact to RDRAND and RDSEED
 
+       srcutree.big_cpu_lim [KNL]
+                       Specifies the number of CPUs constituting a
+                       large system, such that srcu_struct structures
+                       should immediately allocate an srcu_node array.
+                       This kernel-boot parameter defaults to 128,
+                       but takes effect only when the low-order four
+                       bits of srcutree.convert_to_big is equal to 3
+                       (decide at boot).
+
        srcutree.convert_to_big [KNL]
                        Specifies under what conditions an SRCU tree
                        srcu_struct structure will be converted to big
@@ -5616,6 +5625,7 @@
                                   0:  Never.
                                   1:  At init_srcu_struct() time.
                                   2:  When rcutorture decides to.
+                                  3:  Decide at boot time (default).
                                0x1X:  Above plus if high contention.
 
                        Either way, the srcu_node tree will be sized based

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 0bc6a0a..b9dec26 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -41,10 +41,10 @@ module_param(counter_wrap_check, ulong, 0444);
 
 /*
  * Control conversion to SRCU_SIZE_BIG:
- *    0: Don't convert at all (default).
+ *    0: Don't convert at all.
  *    1: Convert at init_srcu_struct() time.
  *    2: Convert when rcutorture invokes srcu_torture_stats_print().
- *    3: Decide at boot time based on system shape.
+ *    3: Decide at boot time based on system shape (default).
  * 0x1x: Convert when excessive contention encountered.
  */
 #define SRCU_SIZING_NONE       0
@@ -57,9 +57,13 @@ module_param(counter_wrap_check, ulong, 0444);
 #define SRCU_SIZING_IS_INIT() (SRCU_SIZING_IS(SRCU_SIZING_INIT))
 #define SRCU_SIZING_IS_TORTURE() (SRCU_SIZING_IS(SRCU_SIZING_TORTURE))
 #define SRCU_SIZING_IS_CONTEND() (convert_to_big & SRCU_SIZING_CONTEND)
-static int convert_to_big = SRCU_SIZING_NONE;
+static int convert_to_big = SRCU_SIZING_AUTO;
 module_param(convert_to_big, int, 0444);
 
+/* Number of CPUs to trigger init_srcu_struct()-time transition to big. */
+static int big_cpu_lim __read_mostly = 128;
+module_param(big_cpu_lim, int, 0444);
+
 /* Contention events per jiffy to initiate transition to big. */
 static int small_contention_lim __read_mostly = 100;
 module_param(small_contention_lim, int, 0444);
@@ -1619,6 +1623,17 @@ void __init srcu_init(void)
 {
        struct srcu_struct *ssp;
 
+       /* Decide on srcu_struct-size strategy. */
+       if (SRCU_SIZING_IS(SRCU_SIZING_AUTO)) {
+               if (nr_cpu_ids >= big_cpu_lim) {
+                       convert_to_big = SRCU_SIZING_INIT; // Don't bother waiting for contention.
+                       pr_info("%s: Setting srcu_struct sizes to big.\n", __func__);
+               } else {
+                       convert_to_big = SRCU_SIZING_NONE | SRCU_SIZING_CONTEND;
+                       pr_info("%s: Setting srcu_struct sizes based on contention.\n", __func__);
+               }
+       }
+
        /*
         * Once that is set, call_srcu() can follow the normal path and
         * queue delayed work. This must follow RCU workqueues creation
@@ -1629,6 +1644,8 @@ void __init srcu_init(void)
                ssp = list_first_entry(&srcu_boot_list, struct srcu_struct,
                                      work.work.entry);
                list_del_init(&ssp->work.work.entry);
+               if (SRCU_SIZING_IS(SRCU_SIZING_INIT) && ssp->srcu_size_state == SRCU_SIZE_SMALL)
+                       ssp->srcu_size_state = SRCU_SIZE_ALLOC;
                queue_work(rcu_gp_wq, &ssp->work.work);
        }
 }