59a0b8aee2a289e0059d40327278ffe4538dcbf4
[platform/kernel/linux-rpi.git] / net / rxrpc / input.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* RxRPC packet reception
3  *
4  * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
5  * Written by David Howells (dhowells@redhat.com)
6  */
7
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10 #include "ar-internal.h"
11
12 static void rxrpc_proto_abort(const char *why,
13                               struct rxrpc_call *call, rxrpc_seq_t seq)
14 {
15         if (rxrpc_abort_call(why, call, seq, RX_PROTOCOL_ERROR, -EBADMSG)) {
16                 set_bit(RXRPC_CALL_EV_ABORT, &call->events);
17                 rxrpc_queue_call(call);
18         }
19 }
20
21 /*
22  * Do TCP-style congestion management [RFC 5681].
23  */
24 static void rxrpc_congestion_management(struct rxrpc_call *call,
25                                         struct sk_buff *skb,
26                                         struct rxrpc_ack_summary *summary,
27                                         rxrpc_serial_t acked_serial)
28 {
29         enum rxrpc_congest_change change = rxrpc_cong_no_change;
30         unsigned int cumulative_acks = call->cong_cumul_acks;
31         unsigned int cwnd = call->cong_cwnd;
32         bool resend = false;
33
34         summary->flight_size =
35                 (call->tx_top - call->tx_hard_ack) - summary->nr_acks;
36
37         if (test_and_clear_bit(RXRPC_CALL_RETRANS_TIMEOUT, &call->flags)) {
38                 summary->retrans_timeo = true;
39                 call->cong_ssthresh = max_t(unsigned int,
40                                             summary->flight_size / 2, 2);
41                 cwnd = 1;
42                 if (cwnd >= call->cong_ssthresh &&
43                     call->cong_mode == RXRPC_CALL_SLOW_START) {
44                         call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
45                         call->cong_tstamp = skb->tstamp;
46                         cumulative_acks = 0;
47                 }
48         }
49
50         cumulative_acks += summary->nr_new_acks;
51         cumulative_acks += summary->nr_rot_new_acks;
52         if (cumulative_acks > 255)
53                 cumulative_acks = 255;
54
55         summary->mode = call->cong_mode;
56         summary->cwnd = call->cong_cwnd;
57         summary->ssthresh = call->cong_ssthresh;
58         summary->cumulative_acks = cumulative_acks;
59         summary->dup_acks = call->cong_dup_acks;
60
61         switch (call->cong_mode) {
62         case RXRPC_CALL_SLOW_START:
63                 if (summary->nr_nacks > 0)
64                         goto packet_loss_detected;
65                 if (summary->cumulative_acks > 0)
66                         cwnd += 1;
67                 if (cwnd >= call->cong_ssthresh) {
68                         call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
69                         call->cong_tstamp = skb->tstamp;
70                 }
71                 goto out;
72
73         case RXRPC_CALL_CONGEST_AVOIDANCE:
74                 if (summary->nr_nacks > 0)
75                         goto packet_loss_detected;
76
77                 /* We analyse the number of packets that get ACK'd per RTT
78                  * period and increase the window if we managed to fill it.
79                  */
80                 if (call->peer->rtt_count == 0)
81                         goto out;
82                 if (ktime_before(skb->tstamp,
83                                  ktime_add_us(call->cong_tstamp,
84                                               call->peer->srtt_us >> 3)))
85                         goto out_no_clear_ca;
86                 change = rxrpc_cong_rtt_window_end;
87                 call->cong_tstamp = skb->tstamp;
88                 if (cumulative_acks >= cwnd)
89                         cwnd++;
90                 goto out;
91
92         case RXRPC_CALL_PACKET_LOSS:
93                 if (summary->nr_nacks == 0)
94                         goto resume_normality;
95
96                 if (summary->new_low_nack) {
97                         change = rxrpc_cong_new_low_nack;
98                         call->cong_dup_acks = 1;
99                         if (call->cong_extra > 1)
100                                 call->cong_extra = 1;
101                         goto send_extra_data;
102                 }
103
104                 call->cong_dup_acks++;
105                 if (call->cong_dup_acks < 3)
106                         goto send_extra_data;
107
108                 change = rxrpc_cong_begin_retransmission;
109                 call->cong_mode = RXRPC_CALL_FAST_RETRANSMIT;
110                 call->cong_ssthresh = max_t(unsigned int,
111                                             summary->flight_size / 2, 2);
112                 cwnd = call->cong_ssthresh + 3;
113                 call->cong_extra = 0;
114                 call->cong_dup_acks = 0;
115                 resend = true;
116                 goto out;
117
118         case RXRPC_CALL_FAST_RETRANSMIT:
119                 if (!summary->new_low_nack) {
120                         if (summary->nr_new_acks == 0)
121                                 cwnd += 1;
122                         call->cong_dup_acks++;
123                         if (call->cong_dup_acks == 2) {
124                                 change = rxrpc_cong_retransmit_again;
125                                 call->cong_dup_acks = 0;
126                                 resend = true;
127                         }
128                 } else {
129                         change = rxrpc_cong_progress;
130                         cwnd = call->cong_ssthresh;
131                         if (summary->nr_nacks == 0)
132                                 goto resume_normality;
133                 }
134                 goto out;
135
136         default:
137                 BUG();
138                 goto out;
139         }
140
141 resume_normality:
142         change = rxrpc_cong_cleared_nacks;
143         call->cong_dup_acks = 0;
144         call->cong_extra = 0;
145         call->cong_tstamp = skb->tstamp;
146         if (cwnd < call->cong_ssthresh)
147                 call->cong_mode = RXRPC_CALL_SLOW_START;
148         else
149                 call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
150 out:
151         cumulative_acks = 0;
152 out_no_clear_ca:
153         if (cwnd >= RXRPC_RXTX_BUFF_SIZE - 1)
154                 cwnd = RXRPC_RXTX_BUFF_SIZE - 1;
155         call->cong_cwnd = cwnd;
156         call->cong_cumul_acks = cumulative_acks;
157         trace_rxrpc_congest(call, summary, acked_serial, change);
158         if (resend && !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))
159                 rxrpc_queue_call(call);
160         return;
161
162 packet_loss_detected:
163         change = rxrpc_cong_saw_nack;
164         call->cong_mode = RXRPC_CALL_PACKET_LOSS;
165         call->cong_dup_acks = 0;
166         goto send_extra_data;
167
168 send_extra_data:
169         /* Send some previously unsent DATA if we have some to advance the ACK
170          * state.
171          */
172         if (call->rxtx_annotations[call->tx_top & RXRPC_RXTX_BUFF_MASK] &
173             RXRPC_TX_ANNO_LAST ||
174             summary->nr_acks != call->tx_top - call->tx_hard_ack) {
175                 call->cong_extra++;
176                 wake_up(&call->waitq);
177         }
178         goto out_no_clear_ca;
179 }
180
181 /*
182  * Apply a hard ACK by advancing the Tx window.
183  */
184 static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to,
185                                    struct rxrpc_ack_summary *summary)
186 {
187         struct sk_buff *skb, *list = NULL;
188         bool rot_last = false;
189         int ix;
190         u8 annotation;
191
192         if (call->acks_lowest_nak == call->tx_hard_ack) {
193                 call->acks_lowest_nak = to;
194         } else if (before_eq(call->acks_lowest_nak, to)) {
195                 summary->new_low_nack = true;
196                 call->acks_lowest_nak = to;
197         }
198
199         spin_lock(&call->lock);
200
201         while (before(call->tx_hard_ack, to)) {
202                 call->tx_hard_ack++;
203                 ix = call->tx_hard_ack & RXRPC_RXTX_BUFF_MASK;
204                 skb = call->rxtx_buffer[ix];
205                 annotation = call->rxtx_annotations[ix];
206                 rxrpc_see_skb(skb, rxrpc_skb_rotated);
207                 call->rxtx_buffer[ix] = NULL;
208                 call->rxtx_annotations[ix] = 0;
209                 skb->next = list;
210                 list = skb;
211
212                 if (annotation & RXRPC_TX_ANNO_LAST) {
213                         set_bit(RXRPC_CALL_TX_LAST, &call->flags);
214                         rot_last = true;
215                 }
216                 if ((annotation & RXRPC_TX_ANNO_MASK) != RXRPC_TX_ANNO_ACK)
217                         summary->nr_rot_new_acks++;
218         }
219
220         spin_unlock(&call->lock);
221
222         trace_rxrpc_transmit(call, (rot_last ?
223                                     rxrpc_transmit_rotate_last :
224                                     rxrpc_transmit_rotate));
225         wake_up(&call->waitq);
226
227         while (list) {
228                 skb = list;
229                 list = skb->next;
230                 skb_mark_not_on_list(skb);
231                 rxrpc_free_skb(skb, rxrpc_skb_freed);
232         }
233
234         return rot_last;
235 }
236
237 /*
238  * End the transmission phase of a call.
239  *
240  * This occurs when we get an ACKALL packet, the first DATA packet of a reply,
241  * or a final ACK packet.
242  */
243 static bool rxrpc_end_tx_phase(struct rxrpc_call *call, bool reply_begun,
244                                const char *abort_why)
245 {
246         unsigned int state;
247
248         ASSERT(test_bit(RXRPC_CALL_TX_LAST, &call->flags));
249
250         write_lock(&call->state_lock);
251
252         state = call->state;
253         switch (state) {
254         case RXRPC_CALL_CLIENT_SEND_REQUEST:
255         case RXRPC_CALL_CLIENT_AWAIT_REPLY:
256                 if (reply_begun)
257                         call->state = state = RXRPC_CALL_CLIENT_RECV_REPLY;
258                 else
259                         call->state = state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
260                 break;
261
262         case RXRPC_CALL_SERVER_AWAIT_ACK:
263                 __rxrpc_call_completed(call);
264                 state = call->state;
265                 break;
266
267         default:
268                 goto bad_state;
269         }
270
271         write_unlock(&call->state_lock);
272         if (state == RXRPC_CALL_CLIENT_AWAIT_REPLY)
273                 trace_rxrpc_transmit(call, rxrpc_transmit_await_reply);
274         else
275                 trace_rxrpc_transmit(call, rxrpc_transmit_end);
276         _leave(" = ok");
277         return true;
278
279 bad_state:
280         write_unlock(&call->state_lock);
281         kdebug("end_tx %s", rxrpc_call_states[call->state]);
282         rxrpc_proto_abort(abort_why, call, call->tx_top);
283         return false;
284 }
285
286 /*
287  * Begin the reply reception phase of a call.
288  */
289 static bool rxrpc_receiving_reply(struct rxrpc_call *call)
290 {
291         struct rxrpc_ack_summary summary = { 0 };
292         unsigned long now, timo;
293         rxrpc_seq_t top = READ_ONCE(call->tx_top);
294
295         if (call->ackr_reason) {
296                 spin_lock_bh(&call->lock);
297                 call->ackr_reason = 0;
298                 spin_unlock_bh(&call->lock);
299                 now = jiffies;
300                 timo = now + MAX_JIFFY_OFFSET;
301                 WRITE_ONCE(call->resend_at, timo);
302                 WRITE_ONCE(call->ack_at, timo);
303                 trace_rxrpc_timer(call, rxrpc_timer_init_for_reply, now);
304         }
305
306         if (!test_bit(RXRPC_CALL_TX_LAST, &call->flags)) {
307                 if (!rxrpc_rotate_tx_window(call, top, &summary)) {
308                         rxrpc_proto_abort("TXL", call, top);
309                         return false;
310                 }
311         }
312         if (!rxrpc_end_tx_phase(call, true, "ETD"))
313                 return false;
314         call->tx_phase = false;
315         return true;
316 }
317
318 /*
319  * Scan a data packet to validate its structure and to work out how many
320  * subpackets it contains.
321  *
322  * A jumbo packet is a collection of consecutive packets glued together with
323  * little headers between that indicate how to change the initial header for
324  * each subpacket.
325  *
326  * RXRPC_JUMBO_PACKET must be set on all but the last subpacket - and all but
327  * the last are RXRPC_JUMBO_DATALEN in size.  The last subpacket may be of any
328  * size.
329  */
330 static bool rxrpc_validate_data(struct sk_buff *skb)
331 {
332         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
333         unsigned int offset = sizeof(struct rxrpc_wire_header);
334         unsigned int len = skb->len;
335         u8 flags = sp->hdr.flags;
336
337         for (;;) {
338                 if (flags & RXRPC_REQUEST_ACK)
339                         __set_bit(sp->nr_subpackets, sp->rx_req_ack);
340                 sp->nr_subpackets++;
341
342                 if (!(flags & RXRPC_JUMBO_PACKET))
343                         break;
344
345                 if (len - offset < RXRPC_JUMBO_SUBPKTLEN)
346                         goto protocol_error;
347                 if (flags & RXRPC_LAST_PACKET)
348                         goto protocol_error;
349                 offset += RXRPC_JUMBO_DATALEN;
350                 if (skb_copy_bits(skb, offset, &flags, 1) < 0)
351                         goto protocol_error;
352                 offset += sizeof(struct rxrpc_jumbo_header);
353         }
354
355         if (flags & RXRPC_LAST_PACKET)
356                 sp->rx_flags |= RXRPC_SKB_INCL_LAST;
357         return true;
358
359 protocol_error:
360         return false;
361 }
362
363 /*
364  * Handle reception of a duplicate packet.
365  *
366  * We have to take care to avoid an attack here whereby we're given a series of
367  * jumbograms, each with a sequence number one before the preceding one and
368  * filled up to maximum UDP size.  If they never send us the first packet in
369  * the sequence, they can cause us to have to hold on to around 2MiB of kernel
370  * space until the call times out.
371  *
372  * We limit the space usage by only accepting three duplicate jumbo packets per
373  * call.  After that, we tell the other side we're no longer accepting jumbos
374  * (that information is encoded in the ACK packet).
375  */
376 static void rxrpc_input_dup_data(struct rxrpc_call *call, rxrpc_seq_t seq,
377                                  bool is_jumbo, bool *_jumbo_bad)
378 {
379         /* Discard normal packets that are duplicates. */
380         if (is_jumbo)
381                 return;
382
383         /* Skip jumbo subpackets that are duplicates.  When we've had three or
384          * more partially duplicate jumbo packets, we refuse to take any more
385          * jumbos for this call.
386          */
387         if (!*_jumbo_bad) {
388                 call->nr_jumbo_bad++;
389                 *_jumbo_bad = true;
390         }
391 }
392
393 /*
394  * Process a DATA packet, adding the packet to the Rx ring.  The caller's
395  * packet ref must be passed on or discarded.
396  */
397 static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb)
398 {
399         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
400         enum rxrpc_call_state state;
401         unsigned int j, nr_subpackets, nr_unacked = 0;
402         rxrpc_serial_t serial = sp->hdr.serial, ack_serial = serial;
403         rxrpc_seq_t seq0 = sp->hdr.seq, hard_ack;
404         bool immediate_ack = false, jumbo_bad = false;
405         u8 ack = 0;
406
407         _enter("{%u,%u},{%u,%u}",
408                call->rx_hard_ack, call->rx_top, skb->len, seq0);
409
410         _proto("Rx DATA %%%u { #%u f=%02x n=%u }",
411                sp->hdr.serial, seq0, sp->hdr.flags, sp->nr_subpackets);
412
413         state = READ_ONCE(call->state);
414         if (state >= RXRPC_CALL_COMPLETE) {
415                 rxrpc_free_skb(skb, rxrpc_skb_freed);
416                 return;
417         }
418
419         if (state == RXRPC_CALL_SERVER_RECV_REQUEST) {
420                 unsigned long timo = READ_ONCE(call->next_req_timo);
421                 unsigned long now, expect_req_by;
422
423                 if (timo) {
424                         now = jiffies;
425                         expect_req_by = now + timo;
426                         WRITE_ONCE(call->expect_req_by, expect_req_by);
427                         rxrpc_reduce_call_timer(call, expect_req_by, now,
428                                                 rxrpc_timer_set_for_idle);
429                 }
430         }
431
432         rxrpc_inc_stat(call->rxnet, stat_rx_data);
433         if (sp->hdr.flags & RXRPC_REQUEST_ACK)
434                 rxrpc_inc_stat(call->rxnet, stat_rx_data_reqack);
435         if (sp->hdr.flags & RXRPC_JUMBO_PACKET)
436                 rxrpc_inc_stat(call->rxnet, stat_rx_data_jumbo);
437
438         spin_lock(&call->input_lock);
439
440         /* Received data implicitly ACKs all of the request packets we sent
441          * when we're acting as a client.
442          */
443         if ((state == RXRPC_CALL_CLIENT_SEND_REQUEST ||
444              state == RXRPC_CALL_CLIENT_AWAIT_REPLY) &&
445             !rxrpc_receiving_reply(call))
446                 goto unlock;
447
448         hard_ack = READ_ONCE(call->rx_hard_ack);
449
450         nr_subpackets = sp->nr_subpackets;
451         if (nr_subpackets > 1) {
452                 if (call->nr_jumbo_bad > 3) {
453                         ack = RXRPC_ACK_NOSPACE;
454                         ack_serial = serial;
455                         goto ack;
456                 }
457         }
458
459         for (j = 0; j < nr_subpackets; j++) {
460                 rxrpc_serial_t serial = sp->hdr.serial + j;
461                 rxrpc_seq_t seq = seq0 + j;
462                 unsigned int ix = seq & RXRPC_RXTX_BUFF_MASK;
463                 bool terminal = (j == nr_subpackets - 1);
464                 bool last = terminal && (sp->rx_flags & RXRPC_SKB_INCL_LAST);
465                 u8 flags, annotation = j;
466
467                 _proto("Rx DATA+%u %%%u { #%x t=%u l=%u }",
468                      j, serial, seq, terminal, last);
469
470                 if (last) {
471                         if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
472                             seq != call->rx_top) {
473                                 rxrpc_proto_abort("LSN", call, seq);
474                                 goto unlock;
475                         }
476                 } else {
477                         if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
478                             after_eq(seq, call->rx_top)) {
479                                 rxrpc_proto_abort("LSA", call, seq);
480                                 goto unlock;
481                         }
482                 }
483
484                 flags = 0;
485                 if (last)
486                         flags |= RXRPC_LAST_PACKET;
487                 if (!terminal)
488                         flags |= RXRPC_JUMBO_PACKET;
489                 if (test_bit(j, sp->rx_req_ack))
490                         flags |= RXRPC_REQUEST_ACK;
491                 trace_rxrpc_rx_data(call->debug_id, seq, serial, flags, annotation);
492
493                 if (before_eq(seq, hard_ack)) {
494                         ack = RXRPC_ACK_DUPLICATE;
495                         ack_serial = serial;
496                         continue;
497                 }
498
499                 if (call->rxtx_buffer[ix]) {
500                         rxrpc_input_dup_data(call, seq, nr_subpackets > 1,
501                                              &jumbo_bad);
502                         if (ack != RXRPC_ACK_DUPLICATE) {
503                                 ack = RXRPC_ACK_DUPLICATE;
504                                 ack_serial = serial;
505                         }
506                         immediate_ack = true;
507                         continue;
508                 }
509
510                 if (after(seq, hard_ack + call->rx_winsize)) {
511                         ack = RXRPC_ACK_EXCEEDS_WINDOW;
512                         ack_serial = serial;
513                         if (flags & RXRPC_JUMBO_PACKET) {
514                                 if (!jumbo_bad) {
515                                         call->nr_jumbo_bad++;
516                                         jumbo_bad = true;
517                                 }
518                         }
519
520                         goto ack;
521                 }
522
523                 if (flags & RXRPC_REQUEST_ACK && !ack) {
524                         ack = RXRPC_ACK_REQUESTED;
525                         ack_serial = serial;
526                 }
527
528                 if (after(seq0, call->ackr_highest_seq))
529                         call->ackr_highest_seq = seq0;
530
531                 /* Queue the packet.  We use a couple of memory barriers here as need
532                  * to make sure that rx_top is perceived to be set after the buffer
533                  * pointer and that the buffer pointer is set after the annotation and
534                  * the skb data.
535                  *
536                  * Barriers against rxrpc_recvmsg_data() and rxrpc_rotate_rx_window()
537                  * and also rxrpc_fill_out_ack().
538                  */
539                 if (!terminal)
540                         rxrpc_get_skb(skb, rxrpc_skb_got);
541                 call->rxtx_annotations[ix] = annotation;
542                 smp_wmb();
543                 call->rxtx_buffer[ix] = skb;
544                 if (after(seq, call->rx_top)) {
545                         smp_store_release(&call->rx_top, seq);
546                 } else if (before(seq, call->rx_top)) {
547                         /* Send an immediate ACK if we fill in a hole */
548                         if (!ack) {
549                                 ack = RXRPC_ACK_DELAY;
550                                 ack_serial = serial;
551                         }
552                         immediate_ack = true;
553                 }
554
555                 if (terminal) {
556                         /* From this point on, we're not allowed to touch the
557                          * packet any longer as its ref now belongs to the Rx
558                          * ring.
559                          */
560                         skb = NULL;
561                         sp = NULL;
562                 }
563
564                 nr_unacked++;
565
566                 if (last) {
567                         set_bit(RXRPC_CALL_RX_LAST, &call->flags);
568                         if (!ack) {
569                                 ack = RXRPC_ACK_DELAY;
570                                 ack_serial = serial;
571                         }
572                         trace_rxrpc_receive(call, rxrpc_receive_queue_last, serial, seq);
573                 } else {
574                         trace_rxrpc_receive(call, rxrpc_receive_queue, serial, seq);
575                 }
576
577                 if (after_eq(seq, call->rx_expect_next)) {
578                         if (after(seq, call->rx_expect_next)) {
579                                 _net("OOS %u > %u", seq, call->rx_expect_next);
580                                 ack = RXRPC_ACK_OUT_OF_SEQUENCE;
581                                 ack_serial = serial;
582                         }
583                         call->rx_expect_next = seq + 1;
584                 }
585                 if (!ack)
586                         ack_serial = serial;
587         }
588
589 ack:
590         if (atomic_add_return(nr_unacked, &call->ackr_nr_unacked) > 2 && !ack)
591                 ack = RXRPC_ACK_IDLE;
592
593         if (ack)
594                 rxrpc_propose_ACK(call, ack, ack_serial,
595                                   immediate_ack, true,
596                                   rxrpc_propose_ack_input_data);
597         else
598                 rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial,
599                                   false, true,
600                                   rxrpc_propose_ack_input_data);
601
602         trace_rxrpc_notify_socket(call->debug_id, serial);
603         rxrpc_notify_socket(call);
604
605 unlock:
606         spin_unlock(&call->input_lock);
607         rxrpc_free_skb(skb, rxrpc_skb_freed);
608         _leave(" [queued]");
609 }
610
611 /*
612  * See if there's a cached RTT probe to complete.
613  */
614 static void rxrpc_complete_rtt_probe(struct rxrpc_call *call,
615                                      ktime_t resp_time,
616                                      rxrpc_serial_t acked_serial,
617                                      rxrpc_serial_t ack_serial,
618                                      enum rxrpc_rtt_rx_trace type)
619 {
620         rxrpc_serial_t orig_serial;
621         unsigned long avail;
622         ktime_t sent_at;
623         bool matched = false;
624         int i;
625
626         avail = READ_ONCE(call->rtt_avail);
627         smp_rmb(); /* Read avail bits before accessing data. */
628
629         for (i = 0; i < ARRAY_SIZE(call->rtt_serial); i++) {
630                 if (!test_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &avail))
631                         continue;
632
633                 sent_at = call->rtt_sent_at[i];
634                 orig_serial = call->rtt_serial[i];
635
636                 if (orig_serial == acked_serial) {
637                         clear_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
638                         smp_mb(); /* Read data before setting avail bit */
639                         set_bit(i, &call->rtt_avail);
640                         if (type != rxrpc_rtt_rx_cancel)
641                                 rxrpc_peer_add_rtt(call, type, i, acked_serial, ack_serial,
642                                                    sent_at, resp_time);
643                         else
644                                 trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_cancel, i,
645                                                    orig_serial, acked_serial, 0, 0);
646                         matched = true;
647                 }
648
649                 /* If a later serial is being acked, then mark this slot as
650                  * being available.
651                  */
652                 if (after(acked_serial, orig_serial)) {
653                         trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_obsolete, i,
654                                            orig_serial, acked_serial, 0, 0);
655                         clear_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
656                         smp_wmb();
657                         set_bit(i, &call->rtt_avail);
658                 }
659         }
660
661         if (!matched)
662                 trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_lost, 9, 0, acked_serial, 0, 0);
663 }
664
665 /*
666  * Process the response to a ping that we sent to find out if we lost an ACK.
667  *
668  * If we got back a ping response that indicates a lower tx_top than what we
669  * had at the time of the ping transmission, we adjudge all the DATA packets
670  * sent between the response tx_top and the ping-time tx_top to have been lost.
671  */
672 static void rxrpc_input_check_for_lost_ack(struct rxrpc_call *call)
673 {
674         rxrpc_seq_t top, bottom, seq;
675         bool resend = false;
676
677         spin_lock_bh(&call->lock);
678
679         bottom = call->tx_hard_ack + 1;
680         top = call->acks_lost_top;
681         if (before(bottom, top)) {
682                 for (seq = bottom; before_eq(seq, top); seq++) {
683                         int ix = seq & RXRPC_RXTX_BUFF_MASK;
684                         u8 annotation = call->rxtx_annotations[ix];
685                         u8 anno_type = annotation & RXRPC_TX_ANNO_MASK;
686
687                         if (anno_type != RXRPC_TX_ANNO_UNACK)
688                                 continue;
689                         annotation &= ~RXRPC_TX_ANNO_MASK;
690                         annotation |= RXRPC_TX_ANNO_RETRANS;
691                         call->rxtx_annotations[ix] = annotation;
692                         resend = true;
693                 }
694         }
695
696         spin_unlock_bh(&call->lock);
697
698         if (resend && !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))
699                 rxrpc_queue_call(call);
700 }
701
702 /*
703  * Process a ping response.
704  */
705 static void rxrpc_input_ping_response(struct rxrpc_call *call,
706                                       ktime_t resp_time,
707                                       rxrpc_serial_t acked_serial,
708                                       rxrpc_serial_t ack_serial)
709 {
710         if (acked_serial == call->acks_lost_ping)
711                 rxrpc_input_check_for_lost_ack(call);
712 }
713
714 /*
715  * Process the extra information that may be appended to an ACK packet
716  */
717 static void rxrpc_input_ackinfo(struct rxrpc_call *call, struct sk_buff *skb,
718                                 struct rxrpc_ackinfo *ackinfo)
719 {
720         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
721         struct rxrpc_peer *peer;
722         unsigned int mtu;
723         bool wake = false;
724         u32 rwind = ntohl(ackinfo->rwind);
725
726         _proto("Rx ACK %%%u Info { rx=%u max=%u rwin=%u jm=%u }",
727                sp->hdr.serial,
728                ntohl(ackinfo->rxMTU), ntohl(ackinfo->maxMTU),
729                rwind, ntohl(ackinfo->jumbo_max));
730
731         if (rwind > RXRPC_RXTX_BUFF_SIZE - 1)
732                 rwind = RXRPC_RXTX_BUFF_SIZE - 1;
733         if (call->tx_winsize != rwind) {
734                 if (rwind > call->tx_winsize)
735                         wake = true;
736                 trace_rxrpc_rx_rwind_change(call, sp->hdr.serial, rwind, wake);
737                 call->tx_winsize = rwind;
738         }
739
740         if (call->cong_ssthresh > rwind)
741                 call->cong_ssthresh = rwind;
742
743         mtu = min(ntohl(ackinfo->rxMTU), ntohl(ackinfo->maxMTU));
744
745         peer = call->peer;
746         if (mtu < peer->maxdata) {
747                 spin_lock_bh(&peer->lock);
748                 peer->maxdata = mtu;
749                 peer->mtu = mtu + peer->hdrsize;
750                 spin_unlock_bh(&peer->lock);
751                 _net("Net MTU %u (maxdata %u)", peer->mtu, peer->maxdata);
752         }
753
754         if (wake)
755                 wake_up(&call->waitq);
756 }
757
758 /*
759  * Process individual soft ACKs.
760  *
761  * Each ACK in the array corresponds to one packet and can be either an ACK or
762  * a NAK.  If we get find an explicitly NAK'd packet we resend immediately;
763  * packets that lie beyond the end of the ACK list are scheduled for resend by
764  * the timer on the basis that the peer might just not have processed them at
765  * the time the ACK was sent.
766  */
767 static void rxrpc_input_soft_acks(struct rxrpc_call *call, u8 *acks,
768                                   rxrpc_seq_t seq, int nr_acks,
769                                   struct rxrpc_ack_summary *summary)
770 {
771         int ix;
772         u8 annotation, anno_type;
773
774         for (; nr_acks > 0; nr_acks--, seq++) {
775                 ix = seq & RXRPC_RXTX_BUFF_MASK;
776                 annotation = call->rxtx_annotations[ix];
777                 anno_type = annotation & RXRPC_TX_ANNO_MASK;
778                 annotation &= ~RXRPC_TX_ANNO_MASK;
779                 switch (*acks++) {
780                 case RXRPC_ACK_TYPE_ACK:
781                         summary->nr_acks++;
782                         if (anno_type == RXRPC_TX_ANNO_ACK)
783                                 continue;
784                         summary->nr_new_acks++;
785                         call->rxtx_annotations[ix] =
786                                 RXRPC_TX_ANNO_ACK | annotation;
787                         break;
788                 case RXRPC_ACK_TYPE_NACK:
789                         if (!summary->nr_nacks &&
790                             call->acks_lowest_nak != seq) {
791                                 call->acks_lowest_nak = seq;
792                                 summary->new_low_nack = true;
793                         }
794                         summary->nr_nacks++;
795                         if (anno_type == RXRPC_TX_ANNO_NAK)
796                                 continue;
797                         summary->nr_new_nacks++;
798                         if (anno_type == RXRPC_TX_ANNO_RETRANS)
799                                 continue;
800                         call->rxtx_annotations[ix] =
801                                 RXRPC_TX_ANNO_NAK | annotation;
802                         break;
803                 default:
804                         return rxrpc_proto_abort("SFT", call, 0);
805                 }
806         }
807 }
808
809 /*
810  * Return true if the ACK is valid - ie. it doesn't appear to have regressed
811  * with respect to the ack state conveyed by preceding ACKs.
812  */
813 static bool rxrpc_is_ack_valid(struct rxrpc_call *call,
814                                rxrpc_seq_t first_pkt, rxrpc_seq_t prev_pkt)
815 {
816         rxrpc_seq_t base = READ_ONCE(call->acks_first_seq);
817
818         if (after(first_pkt, base))
819                 return true; /* The window advanced */
820
821         if (before(first_pkt, base))
822                 return false; /* firstPacket regressed */
823
824         if (after_eq(prev_pkt, call->acks_prev_seq))
825                 return true; /* previousPacket hasn't regressed. */
826
827         /* Some rx implementations put a serial number in previousPacket. */
828         if (after_eq(prev_pkt, base + call->tx_winsize))
829                 return false;
830         return true;
831 }
832
833 /*
834  * Process an ACK packet.
835  *
836  * ack.firstPacket is the sequence number of the first soft-ACK'd/NAK'd packet
837  * in the ACK array.  Anything before that is hard-ACK'd and may be discarded.
838  *
839  * A hard-ACK means that a packet has been processed and may be discarded; a
840  * soft-ACK means that the packet may be discarded and retransmission
841  * requested.  A phase is complete when all packets are hard-ACK'd.
842  */
843 static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
844 {
845         struct rxrpc_ack_summary summary = { 0 };
846         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
847         union {
848                 struct rxrpc_ackpacket ack;
849                 struct rxrpc_ackinfo info;
850                 u8 acks[RXRPC_MAXACKS];
851         } buf;
852         rxrpc_serial_t ack_serial, acked_serial;
853         rxrpc_seq_t first_soft_ack, hard_ack, prev_pkt;
854         int nr_acks, offset, ioffset;
855
856         _enter("");
857
858         offset = sizeof(struct rxrpc_wire_header);
859         if (skb_copy_bits(skb, offset, &buf.ack, sizeof(buf.ack)) < 0) {
860                 _debug("extraction failure");
861                 return rxrpc_proto_abort("XAK", call, 0);
862         }
863         offset += sizeof(buf.ack);
864
865         ack_serial = sp->hdr.serial;
866         acked_serial = ntohl(buf.ack.serial);
867         first_soft_ack = ntohl(buf.ack.firstPacket);
868         prev_pkt = ntohl(buf.ack.previousPacket);
869         hard_ack = first_soft_ack - 1;
870         nr_acks = buf.ack.nAcks;
871         summary.ack_reason = (buf.ack.reason < RXRPC_ACK__INVALID ?
872                               buf.ack.reason : RXRPC_ACK__INVALID);
873
874         trace_rxrpc_rx_ack(call, ack_serial, acked_serial,
875                            first_soft_ack, prev_pkt,
876                            summary.ack_reason, nr_acks);
877         rxrpc_inc_stat(call->rxnet, stat_rx_acks[buf.ack.reason]);
878
879         switch (buf.ack.reason) {
880         case RXRPC_ACK_PING_RESPONSE:
881                 rxrpc_input_ping_response(call, skb->tstamp, acked_serial,
882                                           ack_serial);
883                 rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
884                                          rxrpc_rtt_rx_ping_response);
885                 break;
886         case RXRPC_ACK_REQUESTED:
887                 rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
888                                          rxrpc_rtt_rx_requested_ack);
889                 break;
890         default:
891                 if (acked_serial != 0)
892                         rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
893                                                  rxrpc_rtt_rx_cancel);
894                 break;
895         }
896
897         if (buf.ack.reason == RXRPC_ACK_PING) {
898                 _proto("Rx ACK %%%u PING Request", ack_serial);
899                 rxrpc_propose_ACK(call, RXRPC_ACK_PING_RESPONSE,
900                                   ack_serial, true, true,
901                                   rxrpc_propose_ack_respond_to_ping);
902         } else if (sp->hdr.flags & RXRPC_REQUEST_ACK) {
903                 rxrpc_propose_ACK(call, RXRPC_ACK_REQUESTED,
904                                   ack_serial, true, true,
905                                   rxrpc_propose_ack_respond_to_ack);
906         }
907
908         /* If we get an EXCEEDS_WINDOW ACK from the server, it probably
909          * indicates that the client address changed due to NAT.  The server
910          * lost the call because it switched to a different peer.
911          */
912         if (unlikely(buf.ack.reason == RXRPC_ACK_EXCEEDS_WINDOW) &&
913             first_soft_ack == 1 &&
914             prev_pkt == 0 &&
915             rxrpc_is_client_call(call)) {
916                 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
917                                           0, -ENETRESET);
918                 return;
919         }
920
921         /* If we get an OUT_OF_SEQUENCE ACK from the server, that can also
922          * indicate a change of address.  However, we can retransmit the call
923          * if we still have it buffered to the beginning.
924          */
925         if (unlikely(buf.ack.reason == RXRPC_ACK_OUT_OF_SEQUENCE) &&
926             first_soft_ack == 1 &&
927             prev_pkt == 0 &&
928             call->tx_hard_ack == 0 &&
929             rxrpc_is_client_call(call)) {
930                 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
931                                           0, -ENETRESET);
932                 return;
933         }
934
935         /* Discard any out-of-order or duplicate ACKs (outside lock). */
936         if (!rxrpc_is_ack_valid(call, first_soft_ack, prev_pkt)) {
937                 trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial,
938                                            first_soft_ack, call->acks_first_seq,
939                                            prev_pkt, call->acks_prev_seq);
940                 return;
941         }
942
943         buf.info.rxMTU = 0;
944         ioffset = offset + nr_acks + 3;
945         if (skb->len >= ioffset + sizeof(buf.info) &&
946             skb_copy_bits(skb, ioffset, &buf.info, sizeof(buf.info)) < 0)
947                 return rxrpc_proto_abort("XAI", call, 0);
948
949         spin_lock(&call->input_lock);
950
951         /* Discard any out-of-order or duplicate ACKs (inside lock). */
952         if (!rxrpc_is_ack_valid(call, first_soft_ack, prev_pkt)) {
953                 trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial,
954                                            first_soft_ack, call->acks_first_seq,
955                                            prev_pkt, call->acks_prev_seq);
956                 goto out;
957         }
958         call->acks_latest_ts = skb->tstamp;
959
960         call->acks_first_seq = first_soft_ack;
961         call->acks_prev_seq = prev_pkt;
962
963         if (buf.ack.reason != RXRPC_ACK_PING &&
964             after(acked_serial, call->acks_highest_serial))
965                 call->acks_highest_serial = acked_serial;
966
967         /* Parse rwind and mtu sizes if provided. */
968         if (buf.info.rxMTU)
969                 rxrpc_input_ackinfo(call, skb, &buf.info);
970
971         if (first_soft_ack == 0) {
972                 rxrpc_proto_abort("AK0", call, 0);
973                 goto out;
974         }
975
976         /* Ignore ACKs unless we are or have just been transmitting. */
977         switch (READ_ONCE(call->state)) {
978         case RXRPC_CALL_CLIENT_SEND_REQUEST:
979         case RXRPC_CALL_CLIENT_AWAIT_REPLY:
980         case RXRPC_CALL_SERVER_SEND_REPLY:
981         case RXRPC_CALL_SERVER_AWAIT_ACK:
982                 break;
983         default:
984                 goto out;
985         }
986
987         if (before(hard_ack, call->tx_hard_ack) ||
988             after(hard_ack, call->tx_top)) {
989                 rxrpc_proto_abort("AKW", call, 0);
990                 goto out;
991         }
992         if (nr_acks > call->tx_top - hard_ack) {
993                 rxrpc_proto_abort("AKN", call, 0);
994                 goto out;
995         }
996
997         if (after(hard_ack, call->tx_hard_ack)) {
998                 if (rxrpc_rotate_tx_window(call, hard_ack, &summary)) {
999                         rxrpc_end_tx_phase(call, false, "ETA");
1000                         goto out;
1001                 }
1002         }
1003
1004         if (nr_acks > 0) {
1005                 if (skb_copy_bits(skb, offset, buf.acks, nr_acks) < 0) {
1006                         rxrpc_proto_abort("XSA", call, 0);
1007                         goto out;
1008                 }
1009                 rxrpc_input_soft_acks(call, buf.acks, first_soft_ack, nr_acks,
1010                                       &summary);
1011         }
1012
1013         if (call->rxtx_annotations[call->tx_top & RXRPC_RXTX_BUFF_MASK] &
1014             RXRPC_TX_ANNO_LAST &&
1015             summary.nr_acks == call->tx_top - hard_ack &&
1016             rxrpc_is_client_call(call))
1017                 rxrpc_propose_ACK(call, RXRPC_ACK_PING, ack_serial,
1018                                   false, true,
1019                                   rxrpc_propose_ack_ping_for_lost_reply);
1020
1021         rxrpc_congestion_management(call, skb, &summary, acked_serial);
1022 out:
1023         spin_unlock(&call->input_lock);
1024 }
1025
1026 /*
1027  * Process an ACKALL packet.
1028  */
1029 static void rxrpc_input_ackall(struct rxrpc_call *call, struct sk_buff *skb)
1030 {
1031         struct rxrpc_ack_summary summary = { 0 };
1032         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
1033
1034         _proto("Rx ACKALL %%%u", sp->hdr.serial);
1035
1036         spin_lock(&call->input_lock);
1037
1038         if (rxrpc_rotate_tx_window(call, call->tx_top, &summary))
1039                 rxrpc_end_tx_phase(call, false, "ETL");
1040
1041         spin_unlock(&call->input_lock);
1042 }
1043
1044 /*
1045  * Process an ABORT packet directed at a call.
1046  */
1047 static void rxrpc_input_abort(struct rxrpc_call *call, struct sk_buff *skb)
1048 {
1049         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
1050         __be32 wtmp;
1051         u32 abort_code = RX_CALL_DEAD;
1052
1053         _enter("");
1054
1055         if (skb->len >= 4 &&
1056             skb_copy_bits(skb, sizeof(struct rxrpc_wire_header),
1057                           &wtmp, sizeof(wtmp)) >= 0)
1058                 abort_code = ntohl(wtmp);
1059
1060         trace_rxrpc_rx_abort(call, sp->hdr.serial, abort_code);
1061
1062         _proto("Rx ABORT %%%u { %x }", sp->hdr.serial, abort_code);
1063
1064         rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
1065                                   abort_code, -ECONNABORTED);
1066 }
1067
1068 /*
1069  * Process an incoming call packet.
1070  */
1071 static void rxrpc_input_call_packet(struct rxrpc_call *call,
1072                                     struct sk_buff *skb)
1073 {
1074         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
1075         unsigned long timo;
1076
1077         _enter("%p,%p", call, skb);
1078
1079         timo = READ_ONCE(call->next_rx_timo);
1080         if (timo) {
1081                 unsigned long now = jiffies, expect_rx_by;
1082
1083                 expect_rx_by = now + timo;
1084                 WRITE_ONCE(call->expect_rx_by, expect_rx_by);
1085                 rxrpc_reduce_call_timer(call, expect_rx_by, now,
1086                                         rxrpc_timer_set_for_normal);
1087         }
1088
1089         switch (sp->hdr.type) {
1090         case RXRPC_PACKET_TYPE_DATA:
1091                 rxrpc_input_data(call, skb);
1092                 goto no_free;
1093
1094         case RXRPC_PACKET_TYPE_ACK:
1095                 rxrpc_input_ack(call, skb);
1096                 break;
1097
1098         case RXRPC_PACKET_TYPE_BUSY:
1099                 _proto("Rx BUSY %%%u", sp->hdr.serial);
1100
1101                 /* Just ignore BUSY packets from the server; the retry and
1102                  * lifespan timers will take care of business.  BUSY packets
1103                  * from the client don't make sense.
1104                  */
1105                 break;
1106
1107         case RXRPC_PACKET_TYPE_ABORT:
1108                 rxrpc_input_abort(call, skb);
1109                 break;
1110
1111         case RXRPC_PACKET_TYPE_ACKALL:
1112                 rxrpc_input_ackall(call, skb);
1113                 break;
1114
1115         default:
1116                 break;
1117         }
1118
1119         rxrpc_free_skb(skb, rxrpc_skb_freed);
1120 no_free:
1121         _leave("");
1122 }
1123
1124 /*
1125  * Handle a new service call on a channel implicitly completing the preceding
1126  * call on that channel.  This does not apply to client conns.
1127  *
1128  * TODO: If callNumber > call_id + 1, renegotiate security.
1129  */
1130 static void rxrpc_input_implicit_end_call(struct rxrpc_sock *rx,
1131                                           struct rxrpc_connection *conn,
1132                                           struct rxrpc_call *call)
1133 {
1134         switch (READ_ONCE(call->state)) {
1135         case RXRPC_CALL_SERVER_AWAIT_ACK:
1136                 rxrpc_call_completed(call);
1137                 fallthrough;
1138         case RXRPC_CALL_COMPLETE:
1139                 break;
1140         default:
1141                 if (rxrpc_abort_call("IMP", call, 0, RX_CALL_DEAD, -ESHUTDOWN)) {
1142                         set_bit(RXRPC_CALL_EV_ABORT, &call->events);
1143                         rxrpc_queue_call(call);
1144                 }
1145                 trace_rxrpc_improper_term(call);
1146                 break;
1147         }
1148
1149         spin_lock(&rx->incoming_lock);
1150         __rxrpc_disconnect_call(conn, call);
1151         spin_unlock(&rx->incoming_lock);
1152 }
1153
1154 /*
1155  * post connection-level events to the connection
1156  * - this includes challenges, responses, some aborts and call terminal packet
1157  *   retransmission.
1158  */
1159 static void rxrpc_post_packet_to_conn(struct rxrpc_connection *conn,
1160                                       struct sk_buff *skb)
1161 {
1162         _enter("%p,%p", conn, skb);
1163
1164         skb_queue_tail(&conn->rx_queue, skb);
1165         rxrpc_queue_conn(conn);
1166 }
1167
1168 /*
1169  * post endpoint-level events to the local endpoint
1170  * - this includes debug and version messages
1171  */
1172 static void rxrpc_post_packet_to_local(struct rxrpc_local *local,
1173                                        struct sk_buff *skb)
1174 {
1175         _enter("%p,%p", local, skb);
1176
1177         if (rxrpc_get_local_maybe(local)) {
1178                 skb_queue_tail(&local->event_queue, skb);
1179                 rxrpc_queue_local(local);
1180         } else {
1181                 rxrpc_free_skb(skb, rxrpc_skb_freed);
1182         }
1183 }
1184
1185 /*
1186  * put a packet up for transport-level abort
1187  */
1188 static void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb)
1189 {
1190         if (rxrpc_get_local_maybe(local)) {
1191                 skb_queue_tail(&local->reject_queue, skb);
1192                 rxrpc_queue_local(local);
1193         } else {
1194                 rxrpc_free_skb(skb, rxrpc_skb_freed);
1195         }
1196 }
1197
1198 /*
1199  * Extract the wire header from a packet and translate the byte order.
1200  */
1201 static noinline
1202 int rxrpc_extract_header(struct rxrpc_skb_priv *sp, struct sk_buff *skb)
1203 {
1204         struct rxrpc_wire_header whdr;
1205
1206         /* dig out the RxRPC connection details */
1207         if (skb_copy_bits(skb, 0, &whdr, sizeof(whdr)) < 0) {
1208                 trace_rxrpc_rx_eproto(NULL, sp->hdr.serial,
1209                                       tracepoint_string("bad_hdr"));
1210                 return -EBADMSG;
1211         }
1212
1213         memset(sp, 0, sizeof(*sp));
1214         sp->hdr.epoch           = ntohl(whdr.epoch);
1215         sp->hdr.cid             = ntohl(whdr.cid);
1216         sp->hdr.callNumber      = ntohl(whdr.callNumber);
1217         sp->hdr.seq             = ntohl(whdr.seq);
1218         sp->hdr.serial          = ntohl(whdr.serial);
1219         sp->hdr.flags           = whdr.flags;
1220         sp->hdr.type            = whdr.type;
1221         sp->hdr.userStatus      = whdr.userStatus;
1222         sp->hdr.securityIndex   = whdr.securityIndex;
1223         sp->hdr._rsvd           = ntohs(whdr._rsvd);
1224         sp->hdr.serviceId       = ntohs(whdr.serviceId);
1225         return 0;
1226 }
1227
1228 /*
1229  * handle data received on the local endpoint
1230  * - may be called in interrupt context
1231  *
1232  * [!] Note that as this is called from the encap_rcv hook, the socket is not
1233  * held locked by the caller and nothing prevents sk_user_data on the UDP from
1234  * being cleared in the middle of processing this function.
1235  *
1236  * Called with the RCU read lock held from the IP layer via UDP.
1237  */
1238 int rxrpc_input_packet(struct sock *udp_sk, struct sk_buff *skb)
1239 {
1240         struct rxrpc_local *local = rcu_dereference_sk_user_data(udp_sk);
1241         struct rxrpc_connection *conn;
1242         struct rxrpc_channel *chan;
1243         struct rxrpc_call *call = NULL;
1244         struct rxrpc_skb_priv *sp;
1245         struct rxrpc_peer *peer = NULL;
1246         struct rxrpc_sock *rx = NULL;
1247         unsigned int channel;
1248
1249         _enter("%p", udp_sk);
1250
1251         if (unlikely(!local)) {
1252                 kfree_skb(skb);
1253                 return 0;
1254         }
1255         if (skb->tstamp == 0)
1256                 skb->tstamp = ktime_get_real();
1257
1258         rxrpc_new_skb(skb, rxrpc_skb_received);
1259
1260         skb_pull(skb, sizeof(struct udphdr));
1261
1262         /* The UDP protocol already released all skb resources;
1263          * we are free to add our own data there.
1264          */
1265         sp = rxrpc_skb(skb);
1266
1267         /* dig out the RxRPC connection details */
1268         if (rxrpc_extract_header(sp, skb) < 0)
1269                 goto bad_message;
1270
1271         if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS)) {
1272                 static int lose;
1273                 if ((lose++ & 7) == 7) {
1274                         trace_rxrpc_rx_lose(sp);
1275                         rxrpc_free_skb(skb, rxrpc_skb_lost);
1276                         return 0;
1277                 }
1278         }
1279
1280         if (skb->tstamp == 0)
1281                 skb->tstamp = ktime_get_real();
1282         trace_rxrpc_rx_packet(sp);
1283
1284         switch (sp->hdr.type) {
1285         case RXRPC_PACKET_TYPE_VERSION:
1286                 if (rxrpc_to_client(sp))
1287                         goto discard;
1288                 rxrpc_post_packet_to_local(local, skb);
1289                 goto out;
1290
1291         case RXRPC_PACKET_TYPE_BUSY:
1292                 if (rxrpc_to_server(sp))
1293                         goto discard;
1294                 fallthrough;
1295         case RXRPC_PACKET_TYPE_ACK:
1296         case RXRPC_PACKET_TYPE_ACKALL:
1297                 if (sp->hdr.callNumber == 0)
1298                         goto bad_message;
1299                 fallthrough;
1300         case RXRPC_PACKET_TYPE_ABORT:
1301                 break;
1302
1303         case RXRPC_PACKET_TYPE_DATA:
1304                 if (sp->hdr.callNumber == 0 ||
1305                     sp->hdr.seq == 0)
1306                         goto bad_message;
1307                 if (!rxrpc_validate_data(skb))
1308                         goto bad_message;
1309
1310                 /* Unshare the packet so that it can be modified for in-place
1311                  * decryption.
1312                  */
1313                 if (sp->hdr.securityIndex != 0) {
1314                         struct sk_buff *nskb = skb_unshare(skb, GFP_ATOMIC);
1315                         if (!nskb) {
1316                                 rxrpc_eaten_skb(skb, rxrpc_skb_unshared_nomem);
1317                                 goto out;
1318                         }
1319
1320                         if (nskb != skb) {
1321                                 rxrpc_eaten_skb(skb, rxrpc_skb_received);
1322                                 skb = nskb;
1323                                 rxrpc_new_skb(skb, rxrpc_skb_unshared);
1324                                 sp = rxrpc_skb(skb);
1325                         }
1326                 }
1327                 break;
1328
1329         case RXRPC_PACKET_TYPE_CHALLENGE:
1330                 if (rxrpc_to_server(sp))
1331                         goto discard;
1332                 break;
1333         case RXRPC_PACKET_TYPE_RESPONSE:
1334                 if (rxrpc_to_client(sp))
1335                         goto discard;
1336                 break;
1337
1338                 /* Packet types 9-11 should just be ignored. */
1339         case RXRPC_PACKET_TYPE_PARAMS:
1340         case RXRPC_PACKET_TYPE_10:
1341         case RXRPC_PACKET_TYPE_11:
1342                 goto discard;
1343
1344         default:
1345                 _proto("Rx Bad Packet Type %u", sp->hdr.type);
1346                 goto bad_message;
1347         }
1348
1349         if (sp->hdr.serviceId == 0)
1350                 goto bad_message;
1351
1352         if (rxrpc_to_server(sp)) {
1353                 /* Weed out packets to services we're not offering.  Packets
1354                  * that would begin a call are explicitly rejected and the rest
1355                  * are just discarded.
1356                  */
1357                 rx = rcu_dereference(local->service);
1358                 if (!rx || (sp->hdr.serviceId != rx->srx.srx_service &&
1359                             sp->hdr.serviceId != rx->second_service)) {
1360                         if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA &&
1361                             sp->hdr.seq == 1)
1362                                 goto unsupported_service;
1363                         goto discard;
1364                 }
1365         }
1366
1367         conn = rxrpc_find_connection_rcu(local, skb, &peer);
1368         if (conn) {
1369                 if (sp->hdr.securityIndex != conn->security_ix)
1370                         goto wrong_security;
1371
1372                 if (sp->hdr.serviceId != conn->service_id) {
1373                         int old_id;
1374
1375                         if (!test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags))
1376                                 goto reupgrade;
1377                         old_id = cmpxchg(&conn->service_id, conn->params.service_id,
1378                                          sp->hdr.serviceId);
1379
1380                         if (old_id != conn->params.service_id &&
1381                             old_id != sp->hdr.serviceId)
1382                                 goto reupgrade;
1383                 }
1384
1385                 if (sp->hdr.callNumber == 0) {
1386                         /* Connection-level packet */
1387                         _debug("CONN %p {%d}", conn, conn->debug_id);
1388                         rxrpc_post_packet_to_conn(conn, skb);
1389                         goto out;
1390                 }
1391
1392                 if ((int)sp->hdr.serial - (int)conn->hi_serial > 0)
1393                         conn->hi_serial = sp->hdr.serial;
1394
1395                 /* Call-bound packets are routed by connection channel. */
1396                 channel = sp->hdr.cid & RXRPC_CHANNELMASK;
1397                 chan = &conn->channels[channel];
1398
1399                 /* Ignore really old calls */
1400                 if (sp->hdr.callNumber < chan->last_call)
1401                         goto discard;
1402
1403                 if (sp->hdr.callNumber == chan->last_call) {
1404                         if (chan->call ||
1405                             sp->hdr.type == RXRPC_PACKET_TYPE_ABORT)
1406                                 goto discard;
1407
1408                         /* For the previous service call, if completed
1409                          * successfully, we discard all further packets.
1410                          */
1411                         if (rxrpc_conn_is_service(conn) &&
1412                             chan->last_type == RXRPC_PACKET_TYPE_ACK)
1413                                 goto discard;
1414
1415                         /* But otherwise we need to retransmit the final packet
1416                          * from data cached in the connection record.
1417                          */
1418                         if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA)
1419                                 trace_rxrpc_rx_data(chan->call_debug_id,
1420                                                     sp->hdr.seq,
1421                                                     sp->hdr.serial,
1422                                                     sp->hdr.flags, 0);
1423                         rxrpc_post_packet_to_conn(conn, skb);
1424                         goto out;
1425                 }
1426
1427                 call = rcu_dereference(chan->call);
1428
1429                 if (sp->hdr.callNumber > chan->call_id) {
1430                         if (rxrpc_to_client(sp))
1431                                 goto reject_packet;
1432                         if (call)
1433                                 rxrpc_input_implicit_end_call(rx, conn, call);
1434                         call = NULL;
1435                 }
1436
1437                 if (call) {
1438                         if (sp->hdr.serviceId != call->service_id)
1439                                 call->service_id = sp->hdr.serviceId;
1440                         if ((int)sp->hdr.serial - (int)call->rx_serial > 0)
1441                                 call->rx_serial = sp->hdr.serial;
1442                         if (!test_bit(RXRPC_CALL_RX_HEARD, &call->flags))
1443                                 set_bit(RXRPC_CALL_RX_HEARD, &call->flags);
1444                 }
1445         }
1446
1447         if (!call || refcount_read(&call->ref) == 0) {
1448                 if (rxrpc_to_client(sp) ||
1449                     sp->hdr.type != RXRPC_PACKET_TYPE_DATA)
1450                         goto bad_message;
1451                 if (sp->hdr.seq != 1)
1452                         goto discard;
1453                 call = rxrpc_new_incoming_call(local, rx, skb);
1454                 if (!call)
1455                         goto reject_packet;
1456         }
1457
1458         /* Process a call packet; this either discards or passes on the ref
1459          * elsewhere.
1460          */
1461         rxrpc_input_call_packet(call, skb);
1462         goto out;
1463
1464 discard:
1465         rxrpc_free_skb(skb, rxrpc_skb_freed);
1466 out:
1467         trace_rxrpc_rx_done(0, 0);
1468         return 0;
1469
1470 wrong_security:
1471         trace_rxrpc_abort(0, "SEC", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
1472                           RXKADINCONSISTENCY, EBADMSG);
1473         skb->priority = RXKADINCONSISTENCY;
1474         goto post_abort;
1475
1476 unsupported_service:
1477         trace_rxrpc_abort(0, "INV", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
1478                           RX_INVALID_OPERATION, EOPNOTSUPP);
1479         skb->priority = RX_INVALID_OPERATION;
1480         goto post_abort;
1481
1482 reupgrade:
1483         trace_rxrpc_abort(0, "UPG", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
1484                           RX_PROTOCOL_ERROR, EBADMSG);
1485         goto protocol_error;
1486
1487 bad_message:
1488         trace_rxrpc_abort(0, "BAD", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
1489                           RX_PROTOCOL_ERROR, EBADMSG);
1490 protocol_error:
1491         skb->priority = RX_PROTOCOL_ERROR;
1492 post_abort:
1493         skb->mark = RXRPC_SKB_MARK_REJECT_ABORT;
1494 reject_packet:
1495         trace_rxrpc_rx_done(skb->mark, skb->priority);
1496         rxrpc_reject_packet(local, skb);
1497         _leave(" [badmsg]");
1498         return 0;
1499 }