1 // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
6 #include <bpf/bpf_helpers.h>
7 #include <bpf/bpf_endian.h>
13 #define NSEC_PER_SEC 1000000000L
16 #define ETH_P_IP 0x0800
17 #define ETH_P_IPV6 0x86DD
19 #define tcp_flag_word(tp) (((union tcp_word_hdr *)(tp))->words[3])
23 #define IP_OFFSET 0x1fff
30 #define TCPOPT_WINDOW 3
31 #define TCPOPT_SACK_PERM 4
32 #define TCPOPT_TIMESTAMP 8
35 #define TCPOLEN_WINDOW 3
36 #define TCPOLEN_SACK_PERM 2
37 #define TCPOLEN_TIMESTAMP 10
39 #define TCP_TS_HZ 1000
40 #define TS_OPT_WSCALE_MASK 0xf
41 #define TS_OPT_SACK (1 << 4)
42 #define TS_OPT_ECN (1 << 5)
44 #define TSMASK (((__u32)1 << TSBITS) - 1)
45 #define TCP_MAX_WSCALE 14U
47 #define IPV4_MAXLEN 60
50 #define DEFAULT_MSS4 1460
51 #define DEFAULT_MSS6 1440
52 #define DEFAULT_WSCALE 7
53 #define DEFAULT_TTL 64
54 #define MAX_ALLOWED_PORTS 8
57 do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
59 #define __get_unaligned_t(type, ptr) ({ \
60 const struct { type x; } __attribute__((__packed__)) *__pptr = (typeof(__pptr))(ptr); \
64 #define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr))
67 __uint(type, BPF_MAP_TYPE_ARRAY);
70 __uint(max_entries, 2);
71 } values SEC(".maps");
74 __uint(type, BPF_MAP_TYPE_ARRAY);
77 __uint(max_entries, MAX_ALLOWED_PORTS);
78 } allowed_ports SEC(".maps");
80 /* Some symbols defined in net/netfilter/nf_conntrack_bpf.c are unavailable in
81 * vmlinux.h if CONFIG_NF_CONNTRACK=m, so they are redefined locally.
84 struct bpf_ct_opts___local {
90 } __attribute__((preserve_access_index));
92 #define BPF_F_CURRENT_NETNS (-1)
94 extern struct nf_conn *bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx,
95 struct bpf_sock_tuple *bpf_tuple,
97 struct bpf_ct_opts___local *opts,
98 __u32 len_opts) __ksym;
100 extern struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *skb_ctx,
101 struct bpf_sock_tuple *bpf_tuple,
103 struct bpf_ct_opts___local *opts,
104 u32 len_opts) __ksym;
106 extern void bpf_ct_release(struct nf_conn *ct) __ksym;
108 static __always_inline void swap_eth_addr(__u8 *a, __u8 *b)
112 __builtin_memcpy(tmp, a, ETH_ALEN);
113 __builtin_memcpy(a, b, ETH_ALEN);
114 __builtin_memcpy(b, tmp, ETH_ALEN);
117 static __always_inline __u16 csum_fold(__u32 csum)
119 csum = (csum & 0xffff) + (csum >> 16);
120 csum = (csum & 0xffff) + (csum >> 16);
124 static __always_inline __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
125 __u32 len, __u8 proto,
132 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
134 #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
135 s += (proto + len) << 8;
137 #error Unknown endian
139 s = (s & 0xffffffff) + (s >> 32);
140 s = (s & 0xffffffff) + (s >> 32);
142 return csum_fold((__u32)s);
145 static __always_inline __u16 csum_ipv6_magic(const struct in6_addr *saddr,
146 const struct in6_addr *daddr,
147 __u32 len, __u8 proto, __u32 csum)
153 for (i = 0; i < 4; i++)
154 sum += (__u32)saddr->in6_u.u6_addr32[i];
157 for (i = 0; i < 4; i++)
158 sum += (__u32)daddr->in6_u.u6_addr32[i];
160 /* Don't combine additions to avoid 32-bit overflow. */
161 sum += bpf_htonl(len);
162 sum += bpf_htonl(proto);
164 sum = (sum & 0xffffffff) + (sum >> 32);
165 sum = (sum & 0xffffffff) + (sum >> 32);
167 return csum_fold((__u32)sum);
170 static __always_inline __u64 tcp_clock_ns(void)
172 return bpf_ktime_get_ns();
175 static __always_inline __u32 tcp_ns_to_ts(__u64 ns)
177 return ns / (NSEC_PER_SEC / TCP_TS_HZ);
180 static __always_inline __u32 tcp_time_stamp_raw(void)
182 return tcp_ns_to_ts(tcp_clock_ns());
185 struct tcpopt_context {
191 bool option_timestamp;
195 static int tscookie_tcpopt_parse(struct tcpopt_context *ctx)
199 if (ctx->ptr >= ctx->end)
201 if (ctx->ptr >= ctx->data_end)
204 opcode = ctx->ptr[0];
206 if (opcode == TCPOPT_EOL)
208 if (opcode == TCPOPT_NOP) {
213 if (ctx->ptr + 1 >= ctx->end)
215 if (ctx->ptr + 1 >= ctx->data_end)
217 opsize = ctx->ptr[1];
221 if (ctx->ptr + opsize > ctx->end)
226 if (opsize == TCPOLEN_WINDOW && ctx->ptr + TCPOLEN_WINDOW <= ctx->data_end)
227 ctx->wscale = ctx->ptr[2] < TCP_MAX_WSCALE ? ctx->ptr[2] : TCP_MAX_WSCALE;
229 case TCPOPT_TIMESTAMP:
230 if (opsize == TCPOLEN_TIMESTAMP && ctx->ptr + TCPOLEN_TIMESTAMP <= ctx->data_end) {
231 ctx->option_timestamp = true;
232 /* Client's tsval becomes our tsecr. */
233 *ctx->tsecr = get_unaligned((__be32 *)(ctx->ptr + 2));
236 case TCPOPT_SACK_PERM:
237 if (opsize == TCPOLEN_SACK_PERM)
238 ctx->option_sack = true;
247 static int tscookie_tcpopt_parse_batch(__u32 index, void *context)
251 for (i = 0; i < 7; i++)
252 if (tscookie_tcpopt_parse(context))
257 static __always_inline bool tscookie_init(struct tcphdr *tcp_header,
258 __u16 tcp_len, __be32 *tsval,
259 __be32 *tsecr, void *data_end)
261 struct tcpopt_context loop_ctx = {
262 .ptr = (__u8 *)(tcp_header + 1),
263 .end = (__u8 *)tcp_header + tcp_len,
264 .data_end = data_end,
266 .wscale = TS_OPT_WSCALE_MASK,
267 .option_timestamp = false,
268 .option_sack = false,
272 bpf_loop(6, tscookie_tcpopt_parse_batch, &loop_ctx, 0);
274 if (!loop_ctx.option_timestamp)
277 cookie = tcp_time_stamp_raw() & ~TSMASK;
278 cookie |= loop_ctx.wscale & TS_OPT_WSCALE_MASK;
279 if (loop_ctx.option_sack)
280 cookie |= TS_OPT_SACK;
281 if (tcp_header->ece && tcp_header->cwr)
282 cookie |= TS_OPT_ECN;
283 *tsval = bpf_htonl(cookie);
288 static __always_inline void values_get_tcpipopts(__u16 *mss, __u8 *wscale,
289 __u8 *ttl, bool ipv6)
294 value = bpf_map_lookup_elem(&values, &key);
295 if (value && *value != 0) {
297 *mss = (*value >> 32) & 0xffff;
299 *mss = *value & 0xffff;
300 *wscale = (*value >> 16) & 0xf;
301 *ttl = (*value >> 24) & 0xff;
305 *mss = ipv6 ? DEFAULT_MSS6 : DEFAULT_MSS4;
306 *wscale = DEFAULT_WSCALE;
310 static __always_inline void values_inc_synacks(void)
315 value = bpf_map_lookup_elem(&values, &key);
317 __sync_fetch_and_add(value, 1);
320 static __always_inline bool check_port_allowed(__u16 port)
324 for (i = 0; i < MAX_ALLOWED_PORTS; i++) {
328 value = bpf_map_lookup_elem(&allowed_ports, &key);
332 /* 0 is a terminator value. Check it first to avoid matching on
333 * a forbidden port == 0 and returning true.
345 struct header_pointers {
348 struct ipv6hdr *ipv6;
353 static __always_inline int tcp_dissect(void *data, void *data_end,
354 struct header_pointers *hdr)
357 if (hdr->eth + 1 > data_end)
360 switch (bpf_ntohs(hdr->eth->h_proto)) {
364 hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);
365 if (hdr->ipv4 + 1 > data_end)
367 if (hdr->ipv4->ihl * 4 < sizeof(*hdr->ipv4))
369 if (hdr->ipv4->version != 4)
372 if (hdr->ipv4->protocol != IPPROTO_TCP)
375 hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;
380 hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);
381 if (hdr->ipv6 + 1 > data_end)
383 if (hdr->ipv6->version != 6)
386 /* XXX: Extension headers are not supported and could circumvent
387 * XDP SYN flood protection.
389 if (hdr->ipv6->nexthdr != NEXTHDR_TCP)
392 hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);
395 /* XXX: VLANs will circumvent XDP SYN flood protection. */
399 if (hdr->tcp + 1 > data_end)
401 hdr->tcp_len = hdr->tcp->doff * 4;
402 if (hdr->tcp_len < sizeof(*hdr->tcp))
408 static __always_inline int tcp_lookup(void *ctx, struct header_pointers *hdr, bool xdp)
410 struct bpf_ct_opts___local ct_lookup_opts = {
411 .netns_id = BPF_F_CURRENT_NETNS,
412 .l4proto = IPPROTO_TCP,
414 struct bpf_sock_tuple tup = {};
419 /* TCP doesn't normally use fragments, and XDP can't reassemble
422 if ((hdr->ipv4->frag_off & bpf_htons(IP_DF | IP_MF | IP_OFFSET)) != bpf_htons(IP_DF))
425 tup.ipv4.saddr = hdr->ipv4->saddr;
426 tup.ipv4.daddr = hdr->ipv4->daddr;
427 tup.ipv4.sport = hdr->tcp->source;
428 tup.ipv4.dport = hdr->tcp->dest;
429 tup_size = sizeof(tup.ipv4);
430 } else if (hdr->ipv6) {
431 __builtin_memcpy(tup.ipv6.saddr, &hdr->ipv6->saddr, sizeof(tup.ipv6.saddr));
432 __builtin_memcpy(tup.ipv6.daddr, &hdr->ipv6->daddr, sizeof(tup.ipv6.daddr));
433 tup.ipv6.sport = hdr->tcp->source;
434 tup.ipv6.dport = hdr->tcp->dest;
435 tup_size = sizeof(tup.ipv6);
437 /* The verifier can't track that either ipv4 or ipv6 is not
443 ct = bpf_xdp_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
445 ct = bpf_skb_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
447 unsigned long status = ct->status;
450 if (status & IPS_CONFIRMED)
452 } else if (ct_lookup_opts.error != -ENOENT) {
456 /* error == -ENOENT || !(status & IPS_CONFIRMED) */
460 static __always_inline __u8 tcp_mkoptions(__be32 *buf, __be32 *tsopt, __u16 mss,
465 *buf++ = bpf_htonl((TCPOPT_MSS << 24) | (TCPOLEN_MSS << 16) | mss);
470 if (tsopt[0] & bpf_htonl(1 << 4))
471 *buf++ = bpf_htonl((TCPOPT_SACK_PERM << 24) |
472 (TCPOLEN_SACK_PERM << 16) |
473 (TCPOPT_TIMESTAMP << 8) |
476 *buf++ = bpf_htonl((TCPOPT_NOP << 24) |
478 (TCPOPT_TIMESTAMP << 8) |
483 if ((tsopt[0] & bpf_htonl(0xf)) != bpf_htonl(0xf))
484 *buf++ = bpf_htonl((TCPOPT_NOP << 24) |
485 (TCPOPT_WINDOW << 16) |
486 (TCPOLEN_WINDOW << 8) |
492 static __always_inline void tcp_gen_synack(struct tcphdr *tcp_header,
493 __u32 cookie, __be32 *tsopt,
494 __u16 mss, __u8 wscale)
498 tcp_flag_word(tcp_header) = TCP_FLAG_SYN | TCP_FLAG_ACK;
499 if (tsopt && (tsopt[0] & bpf_htonl(1 << 5)))
500 tcp_flag_word(tcp_header) |= TCP_FLAG_ECE;
501 tcp_header->doff = 5; /* doff is part of tcp_flag_word. */
502 swap(tcp_header->source, tcp_header->dest);
503 tcp_header->ack_seq = bpf_htonl(bpf_ntohl(tcp_header->seq) + 1);
504 tcp_header->seq = bpf_htonl(cookie);
505 tcp_header->window = 0;
506 tcp_header->urg_ptr = 0;
507 tcp_header->check = 0; /* Calculate checksum later. */
509 tcp_options = (void *)(tcp_header + 1);
510 tcp_header->doff += tcp_mkoptions(tcp_options, tsopt, mss, wscale);
513 static __always_inline void tcpv4_gen_synack(struct header_pointers *hdr,
514 __u32 cookie, __be32 *tsopt)
520 values_get_tcpipopts(&mss, &wscale, &ttl, false);
522 swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
524 swap(hdr->ipv4->saddr, hdr->ipv4->daddr);
525 hdr->ipv4->check = 0; /* Calculate checksum later. */
528 hdr->ipv4->ttl = ttl;
530 tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
532 hdr->tcp_len = hdr->tcp->doff * 4;
533 hdr->ipv4->tot_len = bpf_htons(sizeof(*hdr->ipv4) + hdr->tcp_len);
536 static __always_inline void tcpv6_gen_synack(struct header_pointers *hdr,
537 __u32 cookie, __be32 *tsopt)
543 values_get_tcpipopts(&mss, &wscale, &ttl, true);
545 swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
547 swap(hdr->ipv6->saddr, hdr->ipv6->daddr);
548 *(__be32 *)hdr->ipv6 = bpf_htonl(0x60000000);
549 hdr->ipv6->hop_limit = ttl;
551 tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
553 hdr->tcp_len = hdr->tcp->doff * 4;
554 hdr->ipv6->payload_len = bpf_htons(hdr->tcp_len);
557 static __always_inline int syncookie_handle_syn(struct header_pointers *hdr,
559 void *data, void *data_end,
562 __u32 old_pkt_size, new_pkt_size;
563 /* Unlike clang 10, clang 11 and 12 generate code that doesn't pass the
564 * BPF verifier if tsopt is not volatile. Volatile forces it to store
565 * the pointer value and use it directly, otherwise tcp_mkoptions is
566 * (mis)compiled like this:
568 * return buf - start;
569 * reg = stored_return_value_of_tscookie_init;
576 * It creates a dead branch where tsopt is assigned NULL, but the
577 * verifier can't prove it's dead and blocks the program.
579 __be32 * volatile tsopt = NULL;
580 __be32 tsopt_buf[2] = {};
585 /* Checksum is not yet verified, but both checksum failure and TCP
586 * header checks return XDP_DROP, so the order doesn't matter.
588 if (hdr->tcp->fin || hdr->tcp->rst)
591 /* Issue SYN cookies on allowed ports, drop SYN packets on blocked
594 if (!check_port_allowed(bpf_ntohs(hdr->tcp->dest)))
598 /* Check the IPv4 and TCP checksums before creating a SYNACK. */
599 value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, hdr->ipv4->ihl * 4, 0);
602 if (csum_fold(value) != 0)
603 return XDP_DROP; /* Bad IPv4 checksum. */
605 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
608 if (csum_tcpudp_magic(hdr->ipv4->saddr, hdr->ipv4->daddr,
609 hdr->tcp_len, IPPROTO_TCP, value) != 0)
610 return XDP_DROP; /* Bad TCP checksum. */
612 ip_len = sizeof(*hdr->ipv4);
614 value = bpf_tcp_raw_gen_syncookie_ipv4(hdr->ipv4, hdr->tcp,
616 } else if (hdr->ipv6) {
617 /* Check the TCP checksum before creating a SYNACK. */
618 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
621 if (csum_ipv6_magic(&hdr->ipv6->saddr, &hdr->ipv6->daddr,
622 hdr->tcp_len, IPPROTO_TCP, value) != 0)
623 return XDP_DROP; /* Bad TCP checksum. */
625 ip_len = sizeof(*hdr->ipv6);
627 value = bpf_tcp_raw_gen_syncookie_ipv6(hdr->ipv6, hdr->tcp,
635 cookie = (__u32)value;
637 if (tscookie_init((void *)hdr->tcp, hdr->tcp_len,
638 &tsopt_buf[0], &tsopt_buf[1], data_end))
641 /* Check that there is enough space for a SYNACK. It also covers
642 * the check that the destination of the __builtin_memmove below
645 if (data + sizeof(*hdr->eth) + ip_len + TCP_MAXLEN > data_end)
649 if (hdr->ipv4->ihl * 4 > sizeof(*hdr->ipv4)) {
650 struct tcphdr *new_tcp_header;
652 new_tcp_header = data + sizeof(*hdr->eth) + sizeof(*hdr->ipv4);
653 __builtin_memmove(new_tcp_header, hdr->tcp, sizeof(*hdr->tcp));
654 hdr->tcp = new_tcp_header;
656 hdr->ipv4->ihl = sizeof(*hdr->ipv4) / 4;
659 tcpv4_gen_synack(hdr, cookie, tsopt);
660 } else if (hdr->ipv6) {
661 tcpv6_gen_synack(hdr, cookie, tsopt);
666 /* Recalculate checksums. */
668 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
672 hdr->tcp->check = csum_tcpudp_magic(hdr->ipv4->saddr,
678 hdr->ipv4->check = 0;
679 value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, sizeof(*hdr->ipv4), 0);
682 hdr->ipv4->check = csum_fold(value);
683 } else if (hdr->ipv6) {
684 hdr->tcp->check = csum_ipv6_magic(&hdr->ipv6->saddr,
693 /* Set the new packet size. */
694 old_pkt_size = data_end - data;
695 new_pkt_size = sizeof(*hdr->eth) + ip_len + hdr->tcp->doff * 4;
697 if (bpf_xdp_adjust_tail(ctx, new_pkt_size - old_pkt_size))
700 if (bpf_skb_change_tail(ctx, new_pkt_size, 0))
704 values_inc_synacks();
709 static __always_inline int syncookie_handle_ack(struct header_pointers *hdr)
717 err = bpf_tcp_raw_check_syncookie_ipv4(hdr->ipv4, hdr->tcp);
719 err = bpf_tcp_raw_check_syncookie_ipv6(hdr->ipv6, hdr->tcp);
728 static __always_inline int syncookie_part1(void *ctx, void *data, void *data_end,
729 struct header_pointers *hdr, bool xdp)
733 ret = tcp_dissect(data, data_end, hdr);
737 ret = tcp_lookup(ctx, hdr, xdp);
741 /* Packet is TCP and doesn't belong to an established connection. */
743 if ((hdr->tcp->syn ^ hdr->tcp->ack) != 1)
746 /* Grow the TCP header to TCP_MAXLEN to be able to pass any hdr->tcp_len
747 * to bpf_tcp_raw_gen_syncookie_ipv{4,6} and pass the verifier.
750 if (bpf_xdp_adjust_tail(ctx, TCP_MAXLEN - hdr->tcp_len))
753 /* Without volatile the verifier throws this error:
754 * R9 32-bit pointer arithmetic prohibited
756 volatile u64 old_len = data_end - data;
758 if (bpf_skb_change_tail(ctx, old_len + TCP_MAXLEN - hdr->tcp_len, 0))
765 static __always_inline int syncookie_part2(void *ctx, void *data, void *data_end,
766 struct header_pointers *hdr, bool xdp)
770 hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);
771 /* IPV4_MAXLEN is needed when calculating checksum.
772 * At least sizeof(struct iphdr) is needed here to access ihl.
774 if ((void *)hdr->ipv4 + IPV4_MAXLEN > data_end)
776 hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;
777 } else if (hdr->ipv6) {
779 hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);
780 hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);
785 if ((void *)hdr->tcp + TCP_MAXLEN > data_end)
788 /* We run out of registers, tcp_len gets spilled to the stack, and the
789 * verifier forgets its min and max values checked above in tcp_dissect.
791 hdr->tcp_len = hdr->tcp->doff * 4;
792 if (hdr->tcp_len < sizeof(*hdr->tcp))
795 return hdr->tcp->syn ? syncookie_handle_syn(hdr, ctx, data, data_end, xdp) :
796 syncookie_handle_ack(hdr);
800 int syncookie_xdp(struct xdp_md *ctx)
802 void *data_end = (void *)(long)ctx->data_end;
803 void *data = (void *)(long)ctx->data;
804 struct header_pointers hdr;
807 ret = syncookie_part1(ctx, data, data_end, &hdr, true);
811 data_end = (void *)(long)ctx->data_end;
812 data = (void *)(long)ctx->data;
814 return syncookie_part2(ctx, data, data_end, &hdr, true);
818 int syncookie_tc(struct __sk_buff *skb)
820 void *data_end = (void *)(long)skb->data_end;
821 void *data = (void *)(long)skb->data;
822 struct header_pointers hdr;
825 ret = syncookie_part1(skb, data, data_end, &hdr, false);
827 return ret == XDP_PASS ? TC_ACT_OK : TC_ACT_SHOT;
829 data_end = (void *)(long)skb->data_end;
830 data = (void *)(long)skb->data;
832 ret = syncookie_part2(skb, data, data_end, &hdr, false);
837 return bpf_redirect(skb->ifindex, 0);
843 char _license[] SEC("license") = "GPL";