net/rxrpc/call_accept.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* incoming call handling
   3  *
   4  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10 #include <linux/module.h>
  11 #include <linux/net.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/errqueue.h>
  14 #include <linux/udp.h>
  15 #include <linux/in.h>
  16 #include <linux/in6.h>
  17 #include <linux/icmp.h>
  18 #include <linux/gfp.h>
  19 #include <linux/circ_buf.h>
  20 #include <net/sock.h>
  21 #include <net/af_rxrpc.h>
  22 #include <net/ip.h>
  23 #include "ar-internal.h"
  24
  25 static void rxrpc_dummy_notify(struct sock *sk, struct rxrpc_call *call,
  26                                unsigned long user_call_ID)
  27 {
  28 }
  29
  30 /*
  31  * Preallocate a single service call, connection and peer and, if possible,
  32  * give them a user ID and attach the user's side of the ID to them.
  33  */
  34 static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx,
  35                                       struct rxrpc_backlog *b,
  36                                       rxrpc_notify_rx_t notify_rx,
  37                                       rxrpc_user_attach_call_t user_attach_call,
  38                                       unsigned long user_call_ID, gfp_t gfp,
  39                                       unsigned int debug_id)
  40 {
  41         struct rxrpc_call *call, *xcall;
  42         struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
  43         struct rb_node *parent, **pp;
  44         int max, tmp;
  45         unsigned int size = RXRPC_BACKLOG_MAX;
  46         unsigned int head, tail, call_head, call_tail;
  47
  48         max = rx->sk.sk_max_ack_backlog;
  49         tmp = rx->sk.sk_ack_backlog;
  50         if (tmp >= max) {
  51                 _leave(" = -ENOBUFS [full %u]", max);
  52                 return -ENOBUFS;
  53         }
  54         max -= tmp;
  55
  56         /* We don't need more conns and peers than we have calls, but on the
  57          * other hand, we shouldn't ever use more peers than conns or conns
  58          * than calls.
  59          */
  60         call_head = b->call_backlog_head;
  61         call_tail = READ_ONCE(b->call_backlog_tail);
  62         tmp = CIRC_CNT(call_head, call_tail, size);
  63         if (tmp >= max) {
  64                 _leave(" = -ENOBUFS [enough %u]", tmp);
  65                 return -ENOBUFS;
  66         }
  67         max = tmp + 1;
  68
  69         head = b->peer_backlog_head;
  70         tail = READ_ONCE(b->peer_backlog_tail);
  71         if (CIRC_CNT(head, tail, size) < max) {
  72                 struct rxrpc_peer *peer;
  73
  74                 peer = rxrpc_alloc_peer(rx->local, gfp, rxrpc_peer_new_prealloc);
  75                 if (!peer)
  76                         return -ENOMEM;
  77                 b->peer_backlog[head] = peer;
  78                 smp_store_release(&b->peer_backlog_head,
  79                                   (head + 1) & (size - 1));
  80         }
  81
  82         head = b->conn_backlog_head;
  83         tail = READ_ONCE(b->conn_backlog_tail);
  84         if (CIRC_CNT(head, tail, size) < max) {
  85                 struct rxrpc_connection *conn;
  86
  87                 conn = rxrpc_prealloc_service_connection(rxnet, gfp);
  88                 if (!conn)
  89                         return -ENOMEM;
  90                 b->conn_backlog[head] = conn;
  91                 smp_store_release(&b->conn_backlog_head,
  92                                   (head + 1) & (size - 1));
  93         }
  94
  95         /* Now it gets complicated, because calls get registered with the
  96          * socket here, with a user ID preassigned by the user.
  97          */
  98         call = rxrpc_alloc_call(rx, gfp, debug_id);
  99         if (!call)
 100                 return -ENOMEM;
 101         call->flags |= (1 << RXRPC_CALL_IS_SERVICE);
 102         call->state = RXRPC_CALL_SERVER_PREALLOC;
 103
 104         trace_rxrpc_call(call->debug_id, refcount_read(&call->ref),
 105                          user_call_ID, rxrpc_call_new_prealloc_service);
 106
 107         write_lock(&rx->call_lock);
 108
 109         /* Check the user ID isn't already in use */
 110         pp = &rx->calls.rb_node;
 111         parent = NULL;
 112         while (*pp) {
 113                 parent = *pp;
 114                 xcall = rb_entry(parent, struct rxrpc_call, sock_node);
 115                 if (user_call_ID < xcall->user_call_ID)
 116                         pp = &(*pp)->rb_left;
 117                 else if (user_call_ID > xcall->user_call_ID)
 118                         pp = &(*pp)->rb_right;
 119                 else
 120                         goto id_in_use;
 121         }
 122
 123         call->user_call_ID = user_call_ID;
 124         call->notify_rx = notify_rx;
 125         if (user_attach_call) {
 126                 rxrpc_get_call(call, rxrpc_call_get_kernel_service);
 127                 user_attach_call(call, user_call_ID);
 128         }
 129
 130         rxrpc_get_call(call, rxrpc_call_get_userid);
 131         rb_link_node(&call->sock_node, parent, pp);
 132         rb_insert_color(&call->sock_node, &rx->calls);
 133         set_bit(RXRPC_CALL_HAS_USERID, &call->flags);
 134
 135         list_add(&call->sock_link, &rx->sock_calls);
 136
 137         write_unlock(&rx->call_lock);
 138
 139         rxnet = call->rxnet;
 140         spin_lock_bh(&rxnet->call_lock);
 141         list_add_tail_rcu(&call->link, &rxnet->calls);
 142         spin_unlock_bh(&rxnet->call_lock);
 143
 144         b->call_backlog[call_head] = call;
 145         smp_store_release(&b->call_backlog_head, (call_head + 1) & (size - 1));
 146         _leave(" = 0 [%d -> %lx]", call->debug_id, user_call_ID);
 147         return 0;
 148
 149 id_in_use:
 150         write_unlock(&rx->call_lock);
 151         rxrpc_cleanup_call(call);
 152         _leave(" = -EBADSLT");
 153         return -EBADSLT;
 154 }
 155
 156 /*
 157  * Allocate the preallocation buffers for incoming service calls.  These must
 158  * be charged manually.
 159  */
 160 int rxrpc_service_prealloc(struct rxrpc_sock *rx, gfp_t gfp)
 161 {
 162         struct rxrpc_backlog *b = rx->backlog;
 163
 164         if (!b) {
 165                 b = kzalloc(sizeof(struct rxrpc_backlog), gfp);
 166                 if (!b)
 167                         return -ENOMEM;
 168                 rx->backlog = b;
 169         }
 170
 171         return 0;
 172 }
 173
 174 /*
 175  * Discard the preallocation on a service.
 176  */
 177 void rxrpc_discard_prealloc(struct rxrpc_sock *rx)
 178 {
 179         struct rxrpc_backlog *b = rx->backlog;
 180         struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
 181         unsigned int size = RXRPC_BACKLOG_MAX, head, tail;
 182
 183         if (!b)
 184                 return;
 185         rx->backlog = NULL;
 186
 187         /* Make sure that there aren't any incoming calls in progress before we
 188          * clear the preallocation buffers.
 189          */
 190         spin_lock_bh(&rx->incoming_lock);
 191         spin_unlock_bh(&rx->incoming_lock);
 192
 193         head = b->peer_backlog_head;
 194         tail = b->peer_backlog_tail;
 195         while (CIRC_CNT(head, tail, size) > 0) {
 196                 struct rxrpc_peer *peer = b->peer_backlog[tail];
 197                 rxrpc_put_local(peer->local, rxrpc_local_put_prealloc_conn);
 198                 kfree(peer);
 199                 tail = (tail + 1) & (size - 1);
 200         }
 201
 202         head = b->conn_backlog_head;
 203         tail = b->conn_backlog_tail;
 204         while (CIRC_CNT(head, tail, size) > 0) {
 205                 struct rxrpc_connection *conn = b->conn_backlog[tail];
 206                 write_lock(&rxnet->conn_lock);
 207                 list_del(&conn->link);
 208                 list_del(&conn->proc_link);
 209                 write_unlock(&rxnet->conn_lock);
 210                 kfree(conn);
 211                 if (atomic_dec_and_test(&rxnet->nr_conns))
 212                         wake_up_var(&rxnet->nr_conns);
 213                 tail = (tail + 1) & (size - 1);
 214         }
 215
 216         head = b->call_backlog_head;
 217         tail = b->call_backlog_tail;
 218         while (CIRC_CNT(head, tail, size) > 0) {
 219                 struct rxrpc_call *call = b->call_backlog[tail];
 220                 rcu_assign_pointer(call->socket, rx);
 221                 if (rx->discard_new_call) {
 222                         _debug("discard %lx", call->user_call_ID);
 223                         rx->discard_new_call(call, call->user_call_ID);
 224                         if (call->notify_rx)
 225                                 call->notify_rx = rxrpc_dummy_notify;
 226                         rxrpc_put_call(call, rxrpc_call_put_kernel);
 227                 }
 228                 rxrpc_call_completed(call);
 229                 rxrpc_release_call(rx, call);
 230                 rxrpc_put_call(call, rxrpc_call_put_discard_prealloc);
 231                 tail = (tail + 1) & (size - 1);
 232         }
 233
 234         kfree(b);
 235 }
 236
 237 /*
 238  * Ping the other end to fill our RTT cache and to retrieve the rwind
 239  * and MTU parameters.
 240  */
 241 static void rxrpc_send_ping(struct rxrpc_call *call, struct sk_buff *skb)
 242 {
 243         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 244         ktime_t now = skb->tstamp;
 245
 246         if (call->peer->rtt_count < 3 ||
 247             ktime_before(ktime_add_ms(call->peer->rtt_last_req, 1000), now))
 248                 rxrpc_send_ACK(call, RXRPC_ACK_PING, sp->hdr.serial,
 249                                rxrpc_propose_ack_ping_for_params);
 250 }
 251
 252 /*
 253  * Allocate a new incoming call from the prealloc pool, along with a connection
 254  * and a peer as necessary.
 255  */
 256 static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx,
 257                                                     struct rxrpc_local *local,
 258                                                     struct rxrpc_peer *peer,
 259                                                     struct rxrpc_connection *conn,
 260                                                     const struct rxrpc_security *sec,
 261                                                     struct sk_buff *skb)
 262 {
 263         struct rxrpc_backlog *b = rx->backlog;
 264         struct rxrpc_call *call;
 265         unsigned short call_head, conn_head, peer_head;
 266         unsigned short call_tail, conn_tail, peer_tail;
 267         unsigned short call_count, conn_count;
 268
 269         /* #calls >= #conns >= #peers must hold true. */
 270         call_head = smp_load_acquire(&b->call_backlog_head);
 271         call_tail = b->call_backlog_tail;
 272         call_count = CIRC_CNT(call_head, call_tail, RXRPC_BACKLOG_MAX);
 273         conn_head = smp_load_acquire(&b->conn_backlog_head);
 274         conn_tail = b->conn_backlog_tail;
 275         conn_count = CIRC_CNT(conn_head, conn_tail, RXRPC_BACKLOG_MAX);
 276         ASSERTCMP(conn_count, >=, call_count);
 277         peer_head = smp_load_acquire(&b->peer_backlog_head);
 278         peer_tail = b->peer_backlog_tail;
 279         ASSERTCMP(CIRC_CNT(peer_head, peer_tail, RXRPC_BACKLOG_MAX), >=,
 280                   conn_count);
 281
 282         if (call_count == 0)
 283                 return NULL;
 284
 285         if (!conn) {
 286                 if (peer && !rxrpc_get_peer_maybe(peer, rxrpc_peer_get_service_conn))
 287                         peer = NULL;
 288                 if (!peer) {
 289                         peer = b->peer_backlog[peer_tail];
 290                         if (rxrpc_extract_addr_from_skb(&peer->srx, skb) < 0)
 291                                 return NULL;
 292                         b->peer_backlog[peer_tail] = NULL;
 293                         smp_store_release(&b->peer_backlog_tail,
 294                                           (peer_tail + 1) &
 295                                           (RXRPC_BACKLOG_MAX - 1));
 296
 297                         rxrpc_new_incoming_peer(rx, local, peer);
 298                 }
 299
 300                 /* Now allocate and set up the connection */
 301                 conn = b->conn_backlog[conn_tail];
 302                 b->conn_backlog[conn_tail] = NULL;
 303                 smp_store_release(&b->conn_backlog_tail,
 304                                   (conn_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
 305                 conn->local = rxrpc_get_local(local, rxrpc_local_get_prealloc_conn);
 306                 conn->peer = peer;
 307                 rxrpc_see_connection(conn, rxrpc_conn_see_new_service_conn);
 308                 rxrpc_new_incoming_connection(rx, conn, sec, skb);
 309         } else {
 310                 rxrpc_get_connection(conn, rxrpc_conn_get_service_conn);
 311                 atomic_inc(&conn->active);
 312         }
 313
 314         /* And now we can allocate and set up a new call */
 315         call = b->call_backlog[call_tail];
 316         b->call_backlog[call_tail] = NULL;
 317         smp_store_release(&b->call_backlog_tail,
 318                           (call_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
 319
 320         rxrpc_see_call(call, rxrpc_call_see_accept);
 321         call->local = rxrpc_get_local(conn->local, rxrpc_local_get_call);
 322         call->conn = conn;
 323         call->security = conn->security;
 324         call->security_ix = conn->security_ix;
 325         call->peer = rxrpc_get_peer(conn->peer, rxrpc_peer_get_accept);
 326         call->dest_srx = peer->srx;
 327         call->cong_ssthresh = call->peer->cong_ssthresh;
 328         call->tx_last_sent = ktime_get_real();
 329         return call;
 330 }
 331
 332 /*
 333  * Set up a new incoming call.  Called in BH context with the RCU read lock
 334  * held.
 335  *
 336  * If this is for a kernel service, when we allocate the call, it will have
 337  * three refs on it: (1) the kernel service, (2) the user_call_ID tree, (3) the
 338  * retainer ref obtained from the backlog buffer.  Prealloc calls for userspace
 339  * services only have the ref from the backlog buffer.  We pass this ref to the
 340  * caller.
 341  *
 342  * If we want to report an error, we mark the skb with the packet type and
 343  * abort code and return NULL.
 344  *
 345  * The call is returned with the user access mutex held and a ref on it.
 346  */
 347 struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local,
 348                                            struct rxrpc_sock *rx,
 349                                            struct sk_buff *skb)
 350 {
 351         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 352         const struct rxrpc_security *sec = NULL;
 353         struct rxrpc_connection *conn;
 354         struct rxrpc_peer *peer = NULL;
 355         struct rxrpc_call *call = NULL;
 356
 357         _enter("");
 358
 359         spin_lock(&rx->incoming_lock);
 360         if (rx->sk.sk_state == RXRPC_SERVER_LISTEN_DISABLED ||
 361             rx->sk.sk_state == RXRPC_CLOSE) {
 362                 trace_rxrpc_abort(0, "CLS", sp->hdr.cid, sp->hdr.callNumber,
 363                                   sp->hdr.seq, RX_INVALID_OPERATION, ESHUTDOWN);
 364                 skb->mark = RXRPC_SKB_MARK_REJECT_ABORT;
 365                 skb->priority = RX_INVALID_OPERATION;
 366                 goto no_call;
 367         }
 368
 369         /* The peer, connection and call may all have sprung into existence due
 370          * to a duplicate packet being handled on another CPU in parallel, so
 371          * we have to recheck the routing.  However, we're now holding
 372          * rx->incoming_lock, so the values should remain stable.
 373          */
 374         conn = rxrpc_find_connection_rcu(local, skb, &peer);
 375
 376         if (!conn) {
 377                 sec = rxrpc_get_incoming_security(rx, skb);
 378                 if (!sec)
 379                         goto no_call;
 380         }
 381
 382         call = rxrpc_alloc_incoming_call(rx, local, peer, conn, sec, skb);
 383         if (!call) {
 384                 skb->mark = RXRPC_SKB_MARK_REJECT_BUSY;
 385                 goto no_call;
 386         }
 387
 388         trace_rxrpc_receive(call, rxrpc_receive_incoming,
 389                             sp->hdr.serial, sp->hdr.seq);
 390
 391         /* Make the call live. */
 392         rxrpc_incoming_call(rx, call, skb);
 393         conn = call->conn;
 394
 395         if (rx->notify_new_call)
 396                 rx->notify_new_call(&rx->sk, call, call->user_call_ID);
 397
 398         spin_lock(&conn->state_lock);
 399         switch (conn->state) {
 400         case RXRPC_CONN_SERVICE_UNSECURED:
 401                 conn->state = RXRPC_CONN_SERVICE_CHALLENGING;
 402                 set_bit(RXRPC_CONN_EV_CHALLENGE, &call->conn->events);
 403                 rxrpc_queue_conn(call->conn, rxrpc_conn_queue_challenge);
 404                 break;
 405
 406         case RXRPC_CONN_SERVICE:
 407                 write_lock(&call->state_lock);
 408                 if (call->state < RXRPC_CALL_COMPLETE)
 409                         call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
 410                 write_unlock(&call->state_lock);
 411                 break;
 412
 413         case RXRPC_CONN_REMOTELY_ABORTED:
 414                 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
 415                                           conn->abort_code, conn->error);
 416                 break;
 417         case RXRPC_CONN_LOCALLY_ABORTED:
 418                 rxrpc_abort_call("CON", call, sp->hdr.seq,
 419                                  conn->abort_code, conn->error);
 420                 break;
 421         default:
 422                 BUG();
 423         }
 424         spin_unlock(&conn->state_lock);
 425         spin_unlock(&rx->incoming_lock);
 426
 427         rxrpc_send_ping(call, skb);
 428
 429         if (hlist_unhashed(&call->error_link)) {
 430                 spin_lock(&call->peer->lock);
 431                 hlist_add_head(&call->error_link, &call->peer->error_targets);
 432                 spin_unlock(&call->peer->lock);
 433         }
 434
 435         _leave(" = %p{%d}", call, call->debug_id);
 436         return call;
 437
 438 no_call:
 439         spin_unlock(&rx->incoming_lock);
 440         _leave(" = NULL [%u]", skb->mark);
 441         return NULL;
 442 }
 443
 444 /*
 445  * Charge up socket with preallocated calls, attaching user call IDs.
 446  */
 447 int rxrpc_user_charge_accept(struct rxrpc_sock *rx, unsigned long user_call_ID)
 448 {
 449         struct rxrpc_backlog *b = rx->backlog;
 450
 451         if (rx->sk.sk_state == RXRPC_CLOSE)
 452                 return -ESHUTDOWN;
 453
 454         return rxrpc_service_prealloc_one(rx, b, NULL, NULL, user_call_ID,
 455                                           GFP_KERNEL,
 456                                           atomic_inc_return(&rxrpc_debug_id));
 457 }
 458
 459 /*
 460  * rxrpc_kernel_charge_accept - Charge up socket with preallocated calls
 461  * @sock: The socket on which to preallocate
 462  * @notify_rx: Event notification function for the call
 463  * @user_attach_call: Func to attach call to user_call_ID
 464  * @user_call_ID: The tag to attach to the preallocated call
 465  * @gfp: The allocation conditions.
 466  * @debug_id: The tracing debug ID.
 467  *
 468  * Charge up the socket with preallocated calls, each with a user ID.  A
 469  * function should be provided to effect the attachment from the user's side.
 470  * The user is given a ref to hold on the call.
 471  *
 472  * Note that the call may be come connected before this function returns.
 473  */
 474 int rxrpc_kernel_charge_accept(struct socket *sock,
 475                                rxrpc_notify_rx_t notify_rx,
 476                                rxrpc_user_attach_call_t user_attach_call,
 477                                unsigned long user_call_ID, gfp_t gfp,
 478                                unsigned int debug_id)
 479 {
 480         struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
 481         struct rxrpc_backlog *b = rx->backlog;
 482
 483         if (sock->sk->sk_state == RXRPC_CLOSE)
 484                 return -ESHUTDOWN;
 485
 486         return rxrpc_service_prealloc_one(rx, b, notify_rx,
 487                                           user_attach_call, user_call_ID,
 488                                           gfp, debug_id);
 489 }
 490 EXPORT_SYMBOL(rxrpc_kernel_charge_accept);