tcp: tp->urg_data is unlikely to be set

Use some unlikely() hints in the fast path.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 313cf64..9175e0d 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -547,7 +547,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
                int target = sock_rcvlowat(sk, 0, INT_MAX);
                u16 urg_data = READ_ONCE(tp->urg_data);
 
-               if (urg_data &&
+               if (unlikely(urg_data) &&
                    READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
                    !sock_flag(sk, SOCK_URGINLINE))
                        target++;
@@ -1633,7 +1633,7 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
 
                        len = skb->len - offset;
                        /* Stop reading if we hit a patch of urgent data */
-                       if (tp->urg_data) {
+                       if (unlikely(tp->urg_data)) {
                                u32 urg_offset = tp->urg_seq - seq;
                                if (urg_offset < len)
                                        len = urg_offset;
@@ -2326,7 +2326,7 @@ static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
                u32 offset;
 
                /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
-               if (tp->urg_data && tp->urg_seq == *seq) {
+               if (unlikely(tp->urg_data) && tp->urg_seq == *seq) {
                        if (copied)
                                break;
                        if (signal_pending(current)) {
@@ -2431,7 +2431,7 @@ found_ok_skb:
                        used = len;
 
                /* Do we have urgent data here? */
-               if (tp->urg_data) {
+               if (unlikely(tp->urg_data)) {
                        u32 urg_offset = tp->urg_seq - *seq;
                        if (urg_offset < used) {
                                if (!urg_offset) {
@@ -2465,7 +2465,7 @@ found_ok_skb:
                tcp_rcv_space_adjust(sk);
 
 skip_copy:
-               if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
+               if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) {
                        WRITE_ONCE(tp->urg_data, 0);
                        tcp_fast_path_check(sk);
                }

diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 5ee07a3..3658b9c 100644 (file)
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -5604,11 +5604,11 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *t
        struct tcp_sock *tp = tcp_sk(sk);
 
        /* Check if we get a new urgent pointer - normally not. */
-       if (th->urg)
+       if (unlikely(th->urg))
                tcp_check_urg(sk, th);
 
        /* Do we wait for any urgent data? - normally not... */
-       if (tp->urg_data == TCP_URG_NOTYET) {
+       if (unlikely(tp->urg_data == TCP_URG_NOTYET)) {
                u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) -
                          th->syn;