tcp: add PLB functionality for TCP

Congestion control algorithms track PLB state and cause the connection
to trigger a path change when either of the 2 conditions is satisfied:

- No packets are in flight and (# consecutive congested rounds >=
  sysctl_tcp_plb_idle_rehash_rounds)
- (# consecutive congested rounds >= sysctl_tcp_plb_rehash_rounds)

A round (RTT) is marked as congested when congestion signal
(ECN ce_ratio) over an RTT is greater than sysctl_tcp_plb_cong_thresh.
In the event of RTO, PLB (via tcp_write_timeout()) triggers a path
change and disables congestion-triggered path changes for random time
between (sysctl_tcp_plb_suspend_rto_sec, 2*sysctl_tcp_plb_suspend_rto_sec)
to avoid hopping onto the "connectivity blackhole". RTO-triggered
path changes can still happen during this cool-off period.

Signed-off-by: Mubashir Adnan Qureshi <mubashirq@google.com>
Signed-off-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: Neal Cardwell <ncardwell@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 14d45661a84d85816b58bb86a77683e27d4447d8..6b814e788f00d2c3a68f8e27e5871bd8fc2dfd59 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -2140,6 +2140,34 @@ extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
 extern void tcp_rack_reo_timeout(struct sock *sk);
 extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
 
+/* tcp_plb.c */
+
+/*
+ * Scaling factor for fractions in PLB. For example, tcp_plb_update_state
+ * expects cong_ratio which represents fraction of traffic that experienced
+ * congestion over a single RTT. In order to avoid floating point operations,
+ * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in.
+ */
+#define TCP_PLB_SCALE 8
+
+/* State for PLB (Protective Load Balancing) for a single TCP connection. */
+struct tcp_plb_state {
+       u8      consec_cong_rounds:5, /* consecutive congested rounds */
+               unused:3;
+       u32     pause_until; /* jiffies32 when PLB can resume rerouting */
+};
+
+static inline void tcp_plb_init(const struct sock *sk,
+                               struct tcp_plb_state *plb)
+{
+       plb->consec_cong_rounds = 0;
+       plb->pause_until = 0;
+}
+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
+                         const int cong_ratio);
+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb);
+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb);
+
 /* At how many usecs into the future should the RTO fire? */
 static inline s64 tcp_rto_delta_us(const struct sock *sk)
 {

diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index bbdd9c44f14e3c199d74ccf6862464d19980115e..af7d2cf490fbd99f1f5c8cf7f3f1831ae828f56a 100644 (file)
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -10,7 +10,7 @@ obj-y     := route.o inetpeer.o protocol.o \
             tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \
             tcp_minisocks.o tcp_cong.o tcp_metrics.o tcp_fastopen.o \
             tcp_rate.o tcp_recovery.o tcp_ulp.o \
-            tcp_offload.o datagram.o raw.o udp.o udplite.o \
+            tcp_offload.o tcp_plb.o datagram.o raw.o udp.o udplite.o \
             udp_offload.o arp.o icmp.o devinet.o af_inet.o igmp.o \
             fib_frontend.o fib_semantics.o fib_trie.o fib_notifier.o \
             inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o \

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 58b838b56c7fc89ebec22809edbba1828af7e60e..ebab9e8b184c3f6a3a25fde3bdd4fb1205449c04 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -3224,7 +3224,7 @@ static int __net_init tcp_sk_init(struct net *net)
        net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
        net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
        /* Default congestion threshold for PLB to mark a round is 50% */
-       net->ipv4.sysctl_tcp_plb_cong_thresh = 128;
+       net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;
 
        /* Reno is always built in */
        if (!net_eq(net, &init_net) &&

diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c
new file mode 100644 (file)
index 0000000..f4ced37
--- /dev/null
+++ b/net/ipv4/tcp_plb.c
@@ -0,0 +1,107 @@
+/* Protective Load Balancing (PLB)
+ *
+ * PLB was designed to reduce link load imbalance across datacenter
+ * switches. PLB is a host-based optimization; it leverages congestion
+ * signals from the transport layer to randomly change the path of the
+ * connection experiencing sustained congestion. PLB prefers to repath
+ * after idle periods to minimize packet reordering. It repaths by
+ * changing the IPv6 Flow Label on the packets of a connection, which
+ * datacenter switches include as part of ECMP/WCMP hashing.
+ *
+ * PLB is described in detail in:
+ *
+ *     Mubashir Adnan Qureshi, Yuchung Cheng, Qianwen Yin, Qiaobin Fu,
+ *     Gautam Kumar, Masoud Moshref, Junhua Yan, Van Jacobson,
+ *     David Wetherall,Abdul Kabbani:
+ *     "PLB: Congestion Signals are Simple and Effective for
+ *      Network Load Balancing"
+ *     In ACM SIGCOMM 2022, Amsterdam Netherlands.
+ *
+ */
+
+#include <net/tcp.h>
+
+/* Called once per round-trip to update PLB state for a connection. */
+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
+                         const int cong_ratio)
+{
+       struct net *net = sock_net(sk);
+
+       if (!READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))
+               return;
+
+       if (cong_ratio >= 0) {
+               if (cong_ratio < READ_ONCE(net->ipv4.sysctl_tcp_plb_cong_thresh))
+                       plb->consec_cong_rounds = 0;
+               else if (plb->consec_cong_rounds <
+                        READ_ONCE(net->ipv4.sysctl_tcp_plb_rehash_rounds))
+                       plb->consec_cong_rounds++;
+       }
+}
+EXPORT_SYMBOL_GPL(tcp_plb_update_state);
+
+/* Check whether recent congestion has been persistent enough to warrant
+ * a load balancing decision that switches the connection to another path.
+ */
+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb)
+{
+       struct net *net = sock_net(sk);
+       u32 max_suspend;
+       bool forced_rehash = false, idle_rehash = false;
+
+       if (!READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))
+               return;
+
+       forced_rehash = plb->consec_cong_rounds >=
+                       READ_ONCE(net->ipv4.sysctl_tcp_plb_rehash_rounds);
+       /* If sender goes idle then we check whether to rehash. */
+       idle_rehash = READ_ONCE(net->ipv4.sysctl_tcp_plb_idle_rehash_rounds) &&
+                     !tcp_sk(sk)->packets_out &&
+                     plb->consec_cong_rounds >=
+                     READ_ONCE(net->ipv4.sysctl_tcp_plb_idle_rehash_rounds);
+
+       if (!forced_rehash && !idle_rehash)
+               return;
+
+       /* Note that tcp_jiffies32 can wrap; we detect wraps by checking for
+        * cases where the max suspension end is before the actual suspension
+        * end. We clear pause_until to 0 to indicate there is no recent
+        * RTO event that constrains PLB rehashing.
+        */
+       max_suspend = 2 * READ_ONCE(net->ipv4.sysctl_tcp_plb_suspend_rto_sec) * HZ;
+       if (plb->pause_until &&
+           (!before(tcp_jiffies32, plb->pause_until) ||
+            before(tcp_jiffies32 + max_suspend, plb->pause_until)))
+               plb->pause_until = 0;
+
+       if (plb->pause_until)
+               return;
+
+       sk_rethink_txhash(sk);
+       plb->consec_cong_rounds = 0;
+}
+EXPORT_SYMBOL_GPL(tcp_plb_check_rehash);
+
+/* Upon RTO, disallow load balancing for a while, to avoid having load
+ * balancing decisions switch traffic to a black-holed path that was
+ * previously avoided with a sk_rethink_txhash() call at RTO time.
+ */
+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb)
+{
+       struct net *net = sock_net(sk);
+       u32 pause;
+
+       if (!READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))
+               return;
+
+       pause = READ_ONCE(net->ipv4.sysctl_tcp_plb_suspend_rto_sec) * HZ;
+       pause += prandom_u32_max(pause);
+       plb->pause_until = tcp_jiffies32 + pause;
+
+       /* Reset PLB state upon RTO, since an RTO causes a sk_rethink_txhash() call
+        * that may switch this connection to a path with completely different
+        * congestion characteristics.
+        */
+       plb->consec_cong_rounds = 0;
+}
+EXPORT_SYMBOL_GPL(tcp_plb_update_state_upon_rto);