{
mptcp_reset_timer(sk);
- if ((!test_bit(MPTCP_SEND_SPACE, &mptcp_sk(sk)->flags) ||
+ if ((test_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags) ||
mptcp_send_head(sk) ||
(inet_sk_state_load(sk) != TCP_ESTABLISHED)))
mptcp_schedule_work(sk);
put_page(dfrag->page);
}
-static bool mptcp_is_writeable(struct mptcp_sock *msk)
-{
- struct mptcp_subflow_context *subflow;
-
- if (!sk_stream_is_writeable((struct sock *)msk))
- return false;
-
- mptcp_for_each_subflow(msk, subflow) {
- if (sk_stream_is_writeable(subflow->tcp_sock))
- return true;
- }
- return false;
-}
-
static void mptcp_clean_una(struct sock *sk)
{
struct mptcp_sock *msk = mptcp_sk(sk);
mptcp_clean_una(sk);
/* Only wake up writers if a subflow is ready */
- if (mptcp_is_writeable(msk)) {
- set_bit(MPTCP_SEND_SPACE, &msk->flags);
- smp_mb__after_atomic();
-
- /* set SEND_SPACE before sk_stream_write_space clears
- * NOSPACE
- */
+ if (sk_stream_is_writeable(sk)) {
+ clear_bit(MPTCP_NOSPACE, &msk->flags);
sk_stream_write_space(sk);
}
}
{
struct mptcp_subflow_context *subflow;
- clear_bit(MPTCP_SEND_SPACE, &msk->flags);
+ set_bit(MPTCP_NOSPACE, &msk->flags);
smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */
mptcp_for_each_subflow(msk, subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
+ bool ssk_writeable = sk_stream_is_writeable(ssk);
struct socket *sock = READ_ONCE(ssk->sk_socket);
+ if (ssk_writeable || !sock)
+ continue;
+
/* enables ssk->write_space() callbacks */
- if (sock)
- set_bit(SOCK_NOSPACE, &sock->flags);
+ set_bit(SOCK_NOSPACE, &sock->flags);
}
+
+ /* mptcp_data_acked() could run just before we set the NOSPACE bit,
+ * so explicitly check for snd_una value
+ */
+ mptcp_clean_una((struct sock *)msk);
}
static bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)
return NULL;
}
-static void ssk_check_wmem(struct mptcp_sock *msk)
-{
- if (unlikely(!mptcp_is_writeable(msk)))
- mptcp_nospace(msk);
-}
-
static void mptcp_push_release(struct sock *sk, struct sock *ssk,
struct mptcp_sendmsg_info *info)
{
wait_for_memory:
mptcp_nospace(msk);
- mptcp_clean_una(sk);
if (mptcp_timer_pending(sk))
mptcp_reset_timer(sk);
ret = sk_stream_wait_memory(sk, &timeo);
mptcp_push_pending(sk, msg->msg_flags);
out:
- ssk_check_wmem(msk);
release_sock(sk);
return copied ? : ret;
}
INIT_LIST_HEAD(&msk->conn_list);
INIT_LIST_HEAD(&msk->join_list);
INIT_LIST_HEAD(&msk->rtx_queue);
- __set_bit(MPTCP_SEND_SPACE, &msk->flags);
INIT_WORK(&msk->work, mptcp_worker);
msk->out_of_order_queue = RB_ROOT;
msk->first_pending = NULL;
return true;
}
-static bool mptcp_memory_free(const struct sock *sk, int wake)
-{
- struct mptcp_sock *msk = mptcp_sk(sk);
-
- return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true;
-}
-
static struct proto mptcp_prot = {
.name = "MPTCP",
.owner = THIS_MODULE,
.sockets_allocated = &mptcp_sockets_allocated,
.memory_allocated = &tcp_memory_allocated,
.memory_pressure = &tcp_memory_pressure,
- .stream_memory_free = mptcp_memory_free,
.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
.sysctl_mem = sysctl_tcp_mem,
0;
}
+static bool __mptcp_check_writeable(struct mptcp_sock *msk)
+{
+ struct sock *sk = (struct sock *)msk;
+ bool mptcp_writable;
+
+ mptcp_clean_una(sk);
+ mptcp_writable = sk_stream_is_writeable(sk);
+ if (!mptcp_writable)
+ mptcp_nospace(msk);
+
+ return mptcp_writable;
+}
+
+static __poll_t mptcp_check_writeable(struct mptcp_sock *msk)
+{
+ struct sock *sk = (struct sock *)msk;
+ __poll_t ret = 0;
+ bool slow;
+
+ if (unlikely(sk->sk_shutdown & SEND_SHUTDOWN))
+ return 0;
+
+ if (sk_stream_is_writeable(sk))
+ return EPOLLOUT | EPOLLWRNORM;
+
+ slow = lock_sock_fast(sk);
+ if (__mptcp_check_writeable(msk))
+ ret = EPOLLOUT | EPOLLWRNORM;
+
+ unlock_sock_fast(sk, slow);
+ return ret;
+}
+
static __poll_t mptcp_poll(struct file *file, struct socket *sock,
struct poll_table_struct *wait)
{
if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) {
mask |= mptcp_check_readable(msk);
- if (test_bit(MPTCP_SEND_SPACE, &msk->flags))
- mask |= EPOLLOUT | EPOLLWRNORM;
+ mask |= mptcp_check_writeable(msk);
}
if (sk->sk_shutdown & RCV_SHUTDOWN)
mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;