ipv6: fix reachability confirmation with proxy_ndp

[platform/kernel/linux-rpi.git] / net / core / sock_reuseport.c

diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c
index 3f00a28..5a16528 100644 (file)
--- a/net/core/sock_reuseport.c
+++ b/net/core/sock_reuseport.c
@@ -21,6 +21,86 @@ static DEFINE_IDA(reuseport_ida);
 static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
                               struct sock_reuseport *reuse, bool bind_inany);
 
+void reuseport_has_conns_set(struct sock *sk)
+{
+       struct sock_reuseport *reuse;
+
+       if (!rcu_access_pointer(sk->sk_reuseport_cb))
+               return;
+
+       spin_lock_bh(&reuseport_lock);
+       reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
+                                         lockdep_is_held(&reuseport_lock));
+       if (likely(reuse))
+               reuse->has_conns = 1;
+       spin_unlock_bh(&reuseport_lock);
+}
+EXPORT_SYMBOL(reuseport_has_conns_set);
+
+static void __reuseport_get_incoming_cpu(struct sock_reuseport *reuse)
+{
+       /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
+       WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu + 1);
+}
+
+static void __reuseport_put_incoming_cpu(struct sock_reuseport *reuse)
+{
+       /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
+       WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu - 1);
+}
+
+static void reuseport_get_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
+{
+       if (sk->sk_incoming_cpu >= 0)
+               __reuseport_get_incoming_cpu(reuse);
+}
+
+static void reuseport_put_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
+{
+       if (sk->sk_incoming_cpu >= 0)
+               __reuseport_put_incoming_cpu(reuse);
+}
+
+void reuseport_update_incoming_cpu(struct sock *sk, int val)
+{
+       struct sock_reuseport *reuse;
+       int old_sk_incoming_cpu;
+
+       if (unlikely(!rcu_access_pointer(sk->sk_reuseport_cb))) {
+               /* Paired with REAE_ONCE() in sk_incoming_cpu_update()
+                * and compute_score().
+                */
+               WRITE_ONCE(sk->sk_incoming_cpu, val);
+               return;
+       }
+
+       spin_lock_bh(&reuseport_lock);
+
+       /* This must be done under reuseport_lock to avoid a race with
+        * reuseport_grow(), which accesses sk->sk_incoming_cpu without
+        * lock_sock() when detaching a shutdown()ed sk.
+        *
+        * Paired with READ_ONCE() in reuseport_select_sock_by_hash().
+        */
+       old_sk_incoming_cpu = sk->sk_incoming_cpu;
+       WRITE_ONCE(sk->sk_incoming_cpu, val);
+
+       reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
+                                         lockdep_is_held(&reuseport_lock));
+
+       /* reuseport_grow() has detached a closed sk. */
+       if (!reuse)
+               goto out;
+
+       if (old_sk_incoming_cpu < 0 && val >= 0)
+               __reuseport_get_incoming_cpu(reuse);
+       else if (old_sk_incoming_cpu >= 0 && val < 0)
+               __reuseport_put_incoming_cpu(reuse);
+
+out:
+       spin_unlock_bh(&reuseport_lock);
+}
+
 static int reuseport_sock_index(struct sock *sk,
                                const struct sock_reuseport *reuse,
                                bool closed)
@@ -48,6 +128,7 @@ static void __reuseport_add_sock(struct sock *sk,
        /* paired with smp_rmb() in reuseport_(select|migrate)_sock() */
        smp_wmb();
        reuse->num_socks++;
+       reuseport_get_incoming_cpu(sk, reuse);
 }
 
 static bool __reuseport_detach_sock(struct sock *sk,
@@ -60,6 +141,7 @@ static bool __reuseport_detach_sock(struct sock *sk,
 
        reuse->socks[i] = reuse->socks[reuse->num_socks - 1];
        reuse->num_socks--;
+       reuseport_put_incoming_cpu(sk, reuse);
 
        return true;
 }
@@ -70,6 +152,7 @@ static void __reuseport_add_closed_sock(struct sock *sk,
        reuse->socks[reuse->max_socks - reuse->num_closed_socks - 1] = sk;
        /* paired with READ_ONCE() in inet_csk_bind_conflict() */
        WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks + 1);
+       reuseport_get_incoming_cpu(sk, reuse);
 }
 
 static bool __reuseport_detach_closed_sock(struct sock *sk,
@@ -83,6 +166,7 @@ static bool __reuseport_detach_closed_sock(struct sock *sk,
        reuse->socks[i] = reuse->socks[reuse->max_socks - reuse->num_closed_socks];
        /* paired with READ_ONCE() in inet_csk_bind_conflict() */
        WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks - 1);
+       reuseport_put_incoming_cpu(sk, reuse);
 
        return true;
 }
@@ -150,6 +234,7 @@ int reuseport_alloc(struct sock *sk, bool bind_inany)
        reuse->bind_inany = bind_inany;
        reuse->socks[0] = sk;
        reuse->num_socks = 1;
+       reuseport_get_incoming_cpu(sk, reuse);
        rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
 
 out:
@@ -193,6 +278,7 @@ static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)
        more_reuse->reuseport_id = reuse->reuseport_id;
        more_reuse->bind_inany = reuse->bind_inany;
        more_reuse->has_conns = reuse->has_conns;
+       more_reuse->incoming_cpu = reuse->incoming_cpu;
 
        memcpy(more_reuse->socks, reuse->socks,
               reuse->num_socks * sizeof(struct sock *));
@@ -387,7 +473,7 @@ void reuseport_stop_listen_sock(struct sock *sk)
                prog = rcu_dereference_protected(reuse->prog,
                                                 lockdep_is_held(&reuseport_lock));
 
-               if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req ||
+               if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req) ||
                    (prog && prog->expected_attach_type == BPF_SK_REUSEPORT_SELECT_OR_MIGRATE)) {
                        /* Migration capable, move sk from the listening section
                         * to the closed section.
@@ -442,18 +528,32 @@ static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,
 static struct sock *reuseport_select_sock_by_hash(struct sock_reuseport *reuse,
                                                  u32 hash, u16 num_socks)
 {
+       struct sock *first_valid_sk = NULL;
        int i, j;
 
        i = j = reciprocal_scale(hash, num_socks);
-       while (reuse->socks[i]->sk_state == TCP_ESTABLISHED) {
+       do {
+               struct sock *sk = reuse->socks[i];
+
+               if (sk->sk_state != TCP_ESTABLISHED) {
+                       /* Paired with WRITE_ONCE() in __reuseport_(get|put)_incoming_cpu(). */
+                       if (!READ_ONCE(reuse->incoming_cpu))
+                               return sk;
+
+                       /* Paired with WRITE_ONCE() in reuseport_update_incoming_cpu(). */
+                       if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
+                               return sk;
+
+                       if (!first_valid_sk)
+                               first_valid_sk = sk;
+               }
+
                i++;
                if (i >= num_socks)
                        i = 0;
-               if (i == j)
-                       return NULL;
-       }
+       } while (i != j);
 
-       return reuse->socks[i];
+       return first_valid_sk;
 }
 
 /**
@@ -545,7 +645,7 @@ struct sock *reuseport_migrate_sock(struct sock *sk,
        hash = migrating_sk->sk_hash;
        prog = rcu_dereference(reuse->prog);
        if (!prog || prog->expected_attach_type != BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) {
-               if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req)
+               if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req))
                        goto select_by_hash;
                goto failure;
        }