net/rxrpc/call_event.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* Management of Tx window, Tx resend, ACKs and out-of-sequence reception
   3  *
   4  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10 #include <linux/module.h>
  11 #include <linux/circ_buf.h>
  12 #include <linux/net.h>
  13 #include <linux/skbuff.h>
  14 #include <linux/slab.h>
  15 #include <linux/udp.h>
  16 #include <net/sock.h>
  17 #include <net/af_rxrpc.h>
  18 #include "ar-internal.h"
  19
  20 /*
  21  * Propose a PING ACK be sent.
  22  */
  23 void rxrpc_propose_ping(struct rxrpc_call *call, u32 serial,
  24                         enum rxrpc_propose_ack_trace why)
  25 {
  26         unsigned long now = jiffies;
  27         unsigned long ping_at = now + rxrpc_idle_ack_delay;
  28
  29         spin_lock_bh(&call->lock);
  30
  31         if (time_before(ping_at, call->ping_at)) {
  32                 rxrpc_inc_stat(call->rxnet, stat_tx_acks[RXRPC_ACK_PING]);
  33                 WRITE_ONCE(call->ping_at, ping_at);
  34                 rxrpc_reduce_call_timer(call, ping_at, now,
  35                                         rxrpc_timer_set_for_ping);
  36                 trace_rxrpc_propose_ack(call, why, RXRPC_ACK_PING, serial,
  37                                         rxrpc_propose_ack_use);
  38         }
  39
  40         spin_unlock_bh(&call->lock);
  41 }
  42
  43 /*
  44  * propose an ACK be sent
  45  */
  46 static void __rxrpc_propose_ACK(struct rxrpc_call *call, u8 ack_reason,
  47                                 u32 serial, enum rxrpc_propose_ack_trace why)
  48 {
  49         enum rxrpc_propose_ack_outcome outcome = rxrpc_propose_ack_use;
  50         unsigned long expiry = rxrpc_soft_ack_delay;
  51         unsigned long now = jiffies, ack_at;
  52         s8 prior = rxrpc_ack_priority[ack_reason];
  53
  54         rxrpc_inc_stat(call->rxnet, stat_tx_acks[ack_reason]);
  55
  56         /* Update DELAY, IDLE, REQUESTED and PING_RESPONSE ACK serial
  57          * numbers, but we don't alter the timeout.
  58          */
  59         _debug("prior %u %u vs %u %u",
  60                ack_reason, prior,
  61                call->ackr_reason, rxrpc_ack_priority[call->ackr_reason]);
  62         if (ack_reason == call->ackr_reason) {
  63                 if (RXRPC_ACK_UPDATEABLE & (1 << ack_reason)) {
  64                         outcome = rxrpc_propose_ack_update;
  65                         call->ackr_serial = serial;
  66                 }
  67         } else if (prior > rxrpc_ack_priority[call->ackr_reason]) {
  68                 call->ackr_reason = ack_reason;
  69                 call->ackr_serial = serial;
  70         } else {
  71                 outcome = rxrpc_propose_ack_subsume;
  72         }
  73
  74         switch (ack_reason) {
  75         case RXRPC_ACK_REQUESTED:
  76                 if (rxrpc_requested_ack_delay < expiry)
  77                         expiry = rxrpc_requested_ack_delay;
  78                 break;
  79
  80         case RXRPC_ACK_DELAY:
  81                 if (rxrpc_soft_ack_delay < expiry)
  82                         expiry = rxrpc_soft_ack_delay;
  83                 break;
  84
  85         case RXRPC_ACK_IDLE:
  86                 if (rxrpc_idle_ack_delay < expiry)
  87                         expiry = rxrpc_idle_ack_delay;
  88                 break;
  89
  90         default:
  91                 WARN_ON(1);
  92                 return;
  93         }
  94
  95
  96         if (call->peer->srtt_us != 0)
  97                 ack_at = usecs_to_jiffies(call->peer->srtt_us >> 3);
  98         else
  99                 ack_at = expiry;
 100
 101         ack_at += READ_ONCE(call->tx_backoff);
 102         ack_at += now;
 103         if (time_before(ack_at, call->ack_at)) {
 104                 WRITE_ONCE(call->ack_at, ack_at);
 105                 rxrpc_reduce_call_timer(call, ack_at, now,
 106                                         rxrpc_timer_set_for_ack);
 107         }
 108
 109         trace_rxrpc_propose_ack(call, why, ack_reason, serial, outcome);
 110 }
 111
 112 /*
 113  * propose an ACK be sent, locking the call structure
 114  */
 115 void rxrpc_propose_ACK(struct rxrpc_call *call, u8 ack_reason, u32 serial,
 116                        enum rxrpc_propose_ack_trace why)
 117 {
 118         spin_lock_bh(&call->lock);
 119         __rxrpc_propose_ACK(call, ack_reason, serial, why);
 120         spin_unlock_bh(&call->lock);
 121 }
 122
 123 /*
 124  * Queue an ACK for immediate transmission.
 125  */
 126 void rxrpc_send_ACK(struct rxrpc_call *call, u8 ack_reason,
 127                     rxrpc_serial_t serial, enum rxrpc_propose_ack_trace why)
 128 {
 129         struct rxrpc_local *local = call->conn->params.local;
 130         struct rxrpc_txbuf *txb;
 131
 132         if (test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))
 133                 return;
 134
 135         rxrpc_inc_stat(call->rxnet, stat_tx_acks[ack_reason]);
 136
 137         txb = rxrpc_alloc_txbuf(call, RXRPC_PACKET_TYPE_ACK,
 138                                 in_softirq() ? GFP_ATOMIC | __GFP_NOWARN : GFP_NOFS);
 139         if (!txb) {
 140                 kleave(" = -ENOMEM");
 141                 return;
 142         }
 143
 144         txb->ack_why            = why;
 145         txb->wire.seq           = 0;
 146         txb->wire.type          = RXRPC_PACKET_TYPE_ACK;
 147         txb->wire.flags         |= RXRPC_SLOW_START_OK;
 148         txb->ack.bufferSpace    = 0;
 149         txb->ack.maxSkew        = 0;
 150         txb->ack.firstPacket    = 0;
 151         txb->ack.previousPacket = 0;
 152         txb->ack.serial         = htonl(serial);
 153         txb->ack.reason         = ack_reason;
 154         txb->ack.nAcks          = 0;
 155
 156         if (!rxrpc_try_get_call(call, rxrpc_call_got)) {
 157                 rxrpc_put_txbuf(txb, rxrpc_txbuf_put_nomem);
 158                 return;
 159         }
 160
 161         spin_lock_bh(&local->ack_tx_lock);
 162         list_add_tail(&txb->tx_link, &local->ack_tx_queue);
 163         spin_unlock_bh(&local->ack_tx_lock);
 164         trace_rxrpc_send_ack(call, why, ack_reason, serial);
 165
 166         if (in_task()) {
 167                 rxrpc_transmit_ack_packets(call->peer->local);
 168         } else {
 169                 rxrpc_get_local(local);
 170                 rxrpc_queue_local(local);
 171         }
 172 }
 173
 174 /*
 175  * Handle congestion being detected by the retransmit timeout.
 176  */
 177 static void rxrpc_congestion_timeout(struct rxrpc_call *call)
 178 {
 179         set_bit(RXRPC_CALL_RETRANS_TIMEOUT, &call->flags);
 180 }
 181
 182 /*
 183  * Perform retransmission of NAK'd and unack'd packets.
 184  */
 185 static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
 186 {
 187         struct sk_buff *skb;
 188         unsigned long resend_at;
 189         rxrpc_seq_t cursor, seq, top;
 190         ktime_t now, max_age, oldest, ack_ts;
 191         int ix;
 192         u8 annotation, anno_type, retrans = 0, unacked = 0;
 193
 194         _enter("{%d,%d}", call->tx_hard_ack, call->tx_top);
 195
 196         now = ktime_get_real();
 197         max_age = ktime_sub_us(now, jiffies_to_usecs(call->peer->rto_j));
 198
 199         spin_lock_bh(&call->lock);
 200
 201         cursor = call->tx_hard_ack;
 202         top = call->tx_top;
 203         ASSERT(before_eq(cursor, top));
 204         if (cursor == top)
 205                 goto out_unlock;
 206
 207         /* Scan the packet list without dropping the lock and decide which of
 208          * the packets in the Tx buffer we're going to resend and what the new
 209          * resend timeout will be.
 210          */
 211         trace_rxrpc_resend(call, (cursor + 1) & RXRPC_RXTX_BUFF_MASK);
 212         oldest = now;
 213         for (seq = cursor + 1; before_eq(seq, top); seq++) {
 214                 ix = seq & RXRPC_RXTX_BUFF_MASK;
 215                 annotation = call->rxtx_annotations[ix];
 216                 anno_type = annotation & RXRPC_TX_ANNO_MASK;
 217                 annotation &= ~RXRPC_TX_ANNO_MASK;
 218                 if (anno_type == RXRPC_TX_ANNO_ACK)
 219                         continue;
 220
 221                 skb = call->rxtx_buffer[ix];
 222                 rxrpc_see_skb(skb, rxrpc_skb_seen);
 223
 224                 if (anno_type == RXRPC_TX_ANNO_UNACK) {
 225                         if (ktime_after(skb->tstamp, max_age)) {
 226                                 if (ktime_before(skb->tstamp, oldest))
 227                                         oldest = skb->tstamp;
 228                                 continue;
 229                         }
 230                         if (!(annotation & RXRPC_TX_ANNO_RESENT))
 231                                 unacked++;
 232                 }
 233
 234                 /* Okay, we need to retransmit a packet. */
 235                 call->rxtx_annotations[ix] = RXRPC_TX_ANNO_RETRANS | annotation;
 236                 retrans++;
 237                 trace_rxrpc_retransmit(call, seq, annotation | anno_type,
 238                                        ktime_to_ns(ktime_sub(skb->tstamp, max_age)));
 239         }
 240
 241         resend_at = nsecs_to_jiffies(ktime_to_ns(ktime_sub(now, oldest)));
 242         resend_at += jiffies + rxrpc_get_rto_backoff(call->peer, retrans);
 243         WRITE_ONCE(call->resend_at, resend_at);
 244
 245         if (unacked)
 246                 rxrpc_congestion_timeout(call);
 247
 248         /* If there was nothing that needed retransmission then it's likely
 249          * that an ACK got lost somewhere.  Send a ping to find out instead of
 250          * retransmitting data.
 251          */
 252         if (!retrans) {
 253                 rxrpc_reduce_call_timer(call, resend_at, now_j,
 254                                         rxrpc_timer_set_for_resend);
 255                 spin_unlock_bh(&call->lock);
 256                 ack_ts = ktime_sub(now, call->acks_latest_ts);
 257                 if (ktime_to_us(ack_ts) < (call->peer->srtt_us >> 3))
 258                         goto out;
 259                 rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
 260                                rxrpc_propose_ack_ping_for_lost_ack);
 261                 goto out;
 262         }
 263
 264         /* Now go through the Tx window and perform the retransmissions.  We
 265          * have to drop the lock for each send.  If an ACK comes in whilst the
 266          * lock is dropped, it may clear some of the retransmission markers for
 267          * packets that it soft-ACKs.
 268          */
 269         for (seq = cursor + 1; before_eq(seq, top); seq++) {
 270                 ix = seq & RXRPC_RXTX_BUFF_MASK;
 271                 annotation = call->rxtx_annotations[ix];
 272                 anno_type = annotation & RXRPC_TX_ANNO_MASK;
 273                 if (anno_type != RXRPC_TX_ANNO_RETRANS)
 274                         continue;
 275
 276                 /* We need to reset the retransmission state, but we need to do
 277                  * so before we drop the lock as a new ACK/NAK may come in and
 278                  * confuse things
 279                  */
 280                 annotation &= ~RXRPC_TX_ANNO_MASK;
 281                 annotation |= RXRPC_TX_ANNO_UNACK | RXRPC_TX_ANNO_RESENT;
 282                 call->rxtx_annotations[ix] = annotation;
 283
 284                 skb = call->rxtx_buffer[ix];
 285                 if (!skb)
 286                         continue;
 287
 288                 rxrpc_get_skb(skb, rxrpc_skb_got);
 289                 spin_unlock_bh(&call->lock);
 290
 291                 rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);
 292                 if (rxrpc_send_data_packet(call, skb, true) < 0) {
 293                         rxrpc_free_skb(skb, rxrpc_skb_freed);
 294                         return;
 295                 }
 296
 297                 if (rxrpc_is_client_call(call))
 298                         rxrpc_expose_client_call(call);
 299
 300                 rxrpc_free_skb(skb, rxrpc_skb_freed);
 301                 spin_lock_bh(&call->lock);
 302                 if (after(call->tx_hard_ack, seq))
 303                         seq = call->tx_hard_ack;
 304         }
 305
 306 out_unlock:
 307         spin_unlock_bh(&call->lock);
 308 out:
 309         _leave("");
 310 }
 311
 312 /*
 313  * Handle retransmission and deferred ACK/abort generation.
 314  */
 315 void rxrpc_process_call(struct work_struct *work)
 316 {
 317         struct rxrpc_call *call =
 318                 container_of(work, struct rxrpc_call, processor);
 319         unsigned long now, next, t;
 320         unsigned int iterations = 0;
 321         rxrpc_serial_t ackr_serial;
 322         u8 ackr_reason;
 323
 324         rxrpc_see_call(call);
 325
 326         //printk("\n--------------------\n");
 327         _enter("{%d,%s,%lx}",
 328                call->debug_id, rxrpc_call_states[call->state], call->events);
 329
 330 recheck_state:
 331         /* Limit the number of times we do this before returning to the manager */
 332         iterations++;
 333         if (iterations > 5)
 334                 goto requeue;
 335
 336         if (test_and_clear_bit(RXRPC_CALL_EV_ABORT, &call->events)) {
 337                 rxrpc_send_abort_packet(call);
 338                 goto recheck_state;
 339         }
 340
 341         if (call->state == RXRPC_CALL_COMPLETE) {
 342                 rxrpc_delete_call_timer(call);
 343                 goto out_put;
 344         }
 345
 346         /* Work out if any timeouts tripped */
 347         now = jiffies;
 348         t = READ_ONCE(call->expect_rx_by);
 349         if (time_after_eq(now, t)) {
 350                 trace_rxrpc_timer(call, rxrpc_timer_exp_normal, now);
 351                 set_bit(RXRPC_CALL_EV_EXPIRED, &call->events);
 352         }
 353
 354         t = READ_ONCE(call->expect_req_by);
 355         if (call->state == RXRPC_CALL_SERVER_RECV_REQUEST &&
 356             time_after_eq(now, t)) {
 357                 trace_rxrpc_timer(call, rxrpc_timer_exp_idle, now);
 358                 set_bit(RXRPC_CALL_EV_EXPIRED, &call->events);
 359         }
 360
 361         t = READ_ONCE(call->expect_term_by);
 362         if (time_after_eq(now, t)) {
 363                 trace_rxrpc_timer(call, rxrpc_timer_exp_hard, now);
 364                 set_bit(RXRPC_CALL_EV_EXPIRED, &call->events);
 365         }
 366
 367         t = READ_ONCE(call->ack_at);
 368         if (time_after_eq(now, t)) {
 369                 trace_rxrpc_timer(call, rxrpc_timer_exp_ack, now);
 370                 cmpxchg(&call->ack_at, t, now + MAX_JIFFY_OFFSET);
 371                 spin_lock_bh(&call->lock);
 372                 ackr_reason = call->ackr_reason;
 373                 ackr_serial = call->ackr_serial;
 374                 call->ackr_reason = 0;
 375                 call->ackr_serial = 0;
 376                 spin_unlock_bh(&call->lock);
 377                 if (ackr_reason)
 378                         rxrpc_send_ACK(call, ackr_reason, ackr_serial,
 379                                        rxrpc_propose_ack_ping_for_lost_ack);
 380         }
 381
 382         t = READ_ONCE(call->ack_lost_at);
 383         if (time_after_eq(now, t)) {
 384                 trace_rxrpc_timer(call, rxrpc_timer_exp_lost_ack, now);
 385                 cmpxchg(&call->ack_lost_at, t, now + MAX_JIFFY_OFFSET);
 386                 set_bit(RXRPC_CALL_EV_ACK_LOST, &call->events);
 387         }
 388
 389         t = READ_ONCE(call->keepalive_at);
 390         if (time_after_eq(now, t)) {
 391                 trace_rxrpc_timer(call, rxrpc_timer_exp_keepalive, now);
 392                 cmpxchg(&call->keepalive_at, t, now + MAX_JIFFY_OFFSET);
 393                 rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
 394                                rxrpc_propose_ack_ping_for_keepalive);
 395         }
 396
 397         t = READ_ONCE(call->ping_at);
 398         if (time_after_eq(now, t)) {
 399                 trace_rxrpc_timer(call, rxrpc_timer_exp_ping, now);
 400                 cmpxchg(&call->ping_at, t, now + MAX_JIFFY_OFFSET);
 401                 rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
 402                                rxrpc_propose_ack_ping_for_keepalive);
 403         }
 404
 405         t = READ_ONCE(call->resend_at);
 406         if (time_after_eq(now, t)) {
 407                 trace_rxrpc_timer(call, rxrpc_timer_exp_resend, now);
 408                 cmpxchg(&call->resend_at, t, now + MAX_JIFFY_OFFSET);
 409                 set_bit(RXRPC_CALL_EV_RESEND, &call->events);
 410         }
 411
 412         /* Process events */
 413         if (test_and_clear_bit(RXRPC_CALL_EV_EXPIRED, &call->events)) {
 414                 if (test_bit(RXRPC_CALL_RX_HEARD, &call->flags) &&
 415                     (int)call->conn->hi_serial - (int)call->rx_serial > 0) {
 416                         trace_rxrpc_call_reset(call);
 417                         rxrpc_abort_call("EXP", call, 0, RX_CALL_DEAD, -ECONNRESET);
 418                 } else {
 419                         rxrpc_abort_call("EXP", call, 0, RX_CALL_TIMEOUT, -ETIME);
 420                 }
 421                 set_bit(RXRPC_CALL_EV_ABORT, &call->events);
 422                 goto recheck_state;
 423         }
 424
 425         if (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events)) {
 426                 call->acks_lost_top = call->tx_top;
 427                 rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
 428                                rxrpc_propose_ack_ping_for_lost_ack);
 429         }
 430
 431         if (test_and_clear_bit(RXRPC_CALL_EV_RESEND, &call->events) &&
 432             call->state != RXRPC_CALL_CLIENT_RECV_REPLY) {
 433                 rxrpc_resend(call, now);
 434                 goto recheck_state;
 435         }
 436
 437         /* Make sure the timer is restarted */
 438         next = call->expect_rx_by;
 439
 440 #define set(T) { t = READ_ONCE(T); if (time_before(t, next)) next = t; }
 441
 442         set(call->expect_req_by);
 443         set(call->expect_term_by);
 444         set(call->ack_at);
 445         set(call->ack_lost_at);
 446         set(call->resend_at);
 447         set(call->keepalive_at);
 448         set(call->ping_at);
 449
 450         now = jiffies;
 451         if (time_after_eq(now, next))
 452                 goto recheck_state;
 453
 454         rxrpc_reduce_call_timer(call, next, now, rxrpc_timer_restart);
 455
 456         /* other events may have been raised since we started checking */
 457         if (call->events && call->state < RXRPC_CALL_COMPLETE)
 458                 goto requeue;
 459
 460 out_put:
 461         rxrpc_put_call(call, rxrpc_call_put);
 462 out:
 463         _leave("");
 464         return;
 465
 466 requeue:
 467         __rxrpc_queue_call(call);
 468         goto out;
 469 }