tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c

   1 // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
   2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
   3
   4 #include "vmlinux.h"
   5
   6 #include <bpf/bpf_helpers.h>
   7 #include <bpf/bpf_endian.h>
   8 #include <asm/errno.h>
   9
  10 #define TC_ACT_OK 0
  11 #define TC_ACT_SHOT 2
  12
  13 #define NSEC_PER_SEC 1000000000L
  14
  15 #define ETH_ALEN 6
  16 #define ETH_P_IP 0x0800
  17 #define ETH_P_IPV6 0x86DD
  18
  19 #define tcp_flag_word(tp) (((union tcp_word_hdr *)(tp))->words[3])
  20
  21 #define IP_DF 0x4000
  22 #define IP_MF 0x2000
  23 #define IP_OFFSET 0x1fff
  24
  25 #define NEXTHDR_TCP 6
  26
  27 #define TCPOPT_NOP 1
  28 #define TCPOPT_EOL 0
  29 #define TCPOPT_MSS 2
  30 #define TCPOPT_WINDOW 3
  31 #define TCPOPT_SACK_PERM 4
  32 #define TCPOPT_TIMESTAMP 8
  33
  34 #define TCPOLEN_MSS 4
  35 #define TCPOLEN_WINDOW 3
  36 #define TCPOLEN_SACK_PERM 2
  37 #define TCPOLEN_TIMESTAMP 10
  38
  39 #define TCP_TS_HZ 1000
  40 #define TS_OPT_WSCALE_MASK 0xf
  41 #define TS_OPT_SACK (1 << 4)
  42 #define TS_OPT_ECN (1 << 5)
  43 #define TSBITS 6
  44 #define TSMASK (((__u32)1 << TSBITS) - 1)
  45 #define TCP_MAX_WSCALE 14U
  46
  47 #define IPV4_MAXLEN 60
  48 #define TCP_MAXLEN 60
  49
  50 #define DEFAULT_MSS4 1460
  51 #define DEFAULT_MSS6 1440
  52 #define DEFAULT_WSCALE 7
  53 #define DEFAULT_TTL 64
  54 #define MAX_ALLOWED_PORTS 8
  55
  56 #define swap(a, b) \
  57         do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
  58
  59 #define __get_unaligned_t(type, ptr) ({                                         \
  60         const struct { type x; } __attribute__((__packed__)) *__pptr = (typeof(__pptr))(ptr); \
  61         __pptr->x;                                                              \
  62 })
  63
  64 #define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr))
  65
  66 struct {
  67         __uint(type, BPF_MAP_TYPE_ARRAY);
  68         __type(key, __u32);
  69         __type(value, __u64);
  70         __uint(max_entries, 2);
  71 } values SEC(".maps");
  72
  73 struct {
  74         __uint(type, BPF_MAP_TYPE_ARRAY);
  75         __type(key, __u32);
  76         __type(value, __u16);
  77         __uint(max_entries, MAX_ALLOWED_PORTS);
  78 } allowed_ports SEC(".maps");
  79
  80 /* Some symbols defined in net/netfilter/nf_conntrack_bpf.c are unavailable in
  81  * vmlinux.h if CONFIG_NF_CONNTRACK=m, so they are redefined locally.
  82  */
  83
  84 struct bpf_ct_opts___local {
  85         s32 netns_id;
  86         s32 error;
  87         u8 l4proto;
  88         u8 dir;
  89         u8 reserved[2];
  90 } __attribute__((preserve_access_index));
  91
  92 #define BPF_F_CURRENT_NETNS (-1)
  93
  94 extern struct nf_conn *bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx,
  95                                          struct bpf_sock_tuple *bpf_tuple,
  96                                          __u32 len_tuple,
  97                                          struct bpf_ct_opts___local *opts,
  98                                          __u32 len_opts) __ksym;
  99
 100 extern struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *skb_ctx,
 101                                          struct bpf_sock_tuple *bpf_tuple,
 102                                          u32 len_tuple,
 103                                          struct bpf_ct_opts___local *opts,
 104                                          u32 len_opts) __ksym;
 105
 106 extern void bpf_ct_release(struct nf_conn *ct) __ksym;
 107
 108 static __always_inline void swap_eth_addr(__u8 *a, __u8 *b)
 109 {
 110         __u8 tmp[ETH_ALEN];
 111
 112         __builtin_memcpy(tmp, a, ETH_ALEN);
 113         __builtin_memcpy(a, b, ETH_ALEN);
 114         __builtin_memcpy(b, tmp, ETH_ALEN);
 115 }
 116
 117 static __always_inline __u16 csum_fold(__u32 csum)
 118 {
 119         csum = (csum & 0xffff) + (csum >> 16);
 120         csum = (csum & 0xffff) + (csum >> 16);
 121         return (__u16)~csum;
 122 }
 123
 124 static __always_inline __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
 125                                                __u32 len, __u8 proto,
 126                                                __u32 csum)
 127 {
 128         __u64 s = csum;
 129
 130         s += (__u32)saddr;
 131         s += (__u32)daddr;
 132 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 133         s += proto + len;
 134 #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 135         s += (proto + len) << 8;
 136 #else
 137 #error Unknown endian
 138 #endif
 139         s = (s & 0xffffffff) + (s >> 32);
 140         s = (s & 0xffffffff) + (s >> 32);
 141
 142         return csum_fold((__u32)s);
 143 }
 144
 145 static __always_inline __u16 csum_ipv6_magic(const struct in6_addr *saddr,
 146                                              const struct in6_addr *daddr,
 147                                              __u32 len, __u8 proto, __u32 csum)
 148 {
 149         __u64 sum = csum;
 150         int i;
 151
 152 #pragma unroll
 153         for (i = 0; i < 4; i++)
 154                 sum += (__u32)saddr->in6_u.u6_addr32[i];
 155
 156 #pragma unroll
 157         for (i = 0; i < 4; i++)
 158                 sum += (__u32)daddr->in6_u.u6_addr32[i];
 159
 160         /* Don't combine additions to avoid 32-bit overflow. */
 161         sum += bpf_htonl(len);
 162         sum += bpf_htonl(proto);
 163
 164         sum = (sum & 0xffffffff) + (sum >> 32);
 165         sum = (sum & 0xffffffff) + (sum >> 32);
 166
 167         return csum_fold((__u32)sum);
 168 }
 169
 170 static __always_inline __u64 tcp_clock_ns(void)
 171 {
 172         return bpf_ktime_get_ns();
 173 }
 174
 175 static __always_inline __u32 tcp_ns_to_ts(__u64 ns)
 176 {
 177         return ns / (NSEC_PER_SEC / TCP_TS_HZ);
 178 }
 179
 180 static __always_inline __u32 tcp_time_stamp_raw(void)
 181 {
 182         return tcp_ns_to_ts(tcp_clock_ns());
 183 }
 184
 185 struct tcpopt_context {
 186         __u8 *ptr;
 187         __u8 *end;
 188         void *data_end;
 189         __be32 *tsecr;
 190         __u8 wscale;
 191         bool option_timestamp;
 192         bool option_sack;
 193 };
 194
 195 static int tscookie_tcpopt_parse(struct tcpopt_context *ctx)
 196 {
 197         __u8 opcode, opsize;
 198
 199         if (ctx->ptr >= ctx->end)
 200                 return 1;
 201         if (ctx->ptr >= ctx->data_end)
 202                 return 1;
 203
 204         opcode = ctx->ptr[0];
 205
 206         if (opcode == TCPOPT_EOL)
 207                 return 1;
 208         if (opcode == TCPOPT_NOP) {
 209                 ++ctx->ptr;
 210                 return 0;
 211         }
 212
 213         if (ctx->ptr + 1 >= ctx->end)
 214                 return 1;
 215         if (ctx->ptr + 1 >= ctx->data_end)
 216                 return 1;
 217         opsize = ctx->ptr[1];
 218         if (opsize < 2)
 219                 return 1;
 220
 221         if (ctx->ptr + opsize > ctx->end)
 222                 return 1;
 223
 224         switch (opcode) {
 225         case TCPOPT_WINDOW:
 226                 if (opsize == TCPOLEN_WINDOW && ctx->ptr + TCPOLEN_WINDOW <= ctx->data_end)
 227                         ctx->wscale = ctx->ptr[2] < TCP_MAX_WSCALE ? ctx->ptr[2] : TCP_MAX_WSCALE;
 228                 break;
 229         case TCPOPT_TIMESTAMP:
 230                 if (opsize == TCPOLEN_TIMESTAMP && ctx->ptr + TCPOLEN_TIMESTAMP <= ctx->data_end) {
 231                         ctx->option_timestamp = true;
 232                         /* Client's tsval becomes our tsecr. */
 233                         *ctx->tsecr = get_unaligned((__be32 *)(ctx->ptr + 2));
 234                 }
 235                 break;
 236         case TCPOPT_SACK_PERM:
 237                 if (opsize == TCPOLEN_SACK_PERM)
 238                         ctx->option_sack = true;
 239                 break;
 240         }
 241
 242         ctx->ptr += opsize;
 243
 244         return 0;
 245 }
 246
 247 static int tscookie_tcpopt_parse_batch(__u32 index, void *context)
 248 {
 249         int i;
 250
 251         for (i = 0; i < 7; i++)
 252                 if (tscookie_tcpopt_parse(context))
 253                         return 1;
 254         return 0;
 255 }
 256
 257 static __always_inline bool tscookie_init(struct tcphdr *tcp_header,
 258                                           __u16 tcp_len, __be32 *tsval,
 259                                           __be32 *tsecr, void *data_end)
 260 {
 261         struct tcpopt_context loop_ctx = {
 262                 .ptr = (__u8 *)(tcp_header + 1),
 263                 .end = (__u8 *)tcp_header + tcp_len,
 264                 .data_end = data_end,
 265                 .tsecr = tsecr,
 266                 .wscale = TS_OPT_WSCALE_MASK,
 267                 .option_timestamp = false,
 268                 .option_sack = false,
 269         };
 270         u32 cookie;
 271
 272         bpf_loop(6, tscookie_tcpopt_parse_batch, &loop_ctx, 0);
 273
 274         if (!loop_ctx.option_timestamp)
 275                 return false;
 276
 277         cookie = tcp_time_stamp_raw() & ~TSMASK;
 278         cookie |= loop_ctx.wscale & TS_OPT_WSCALE_MASK;
 279         if (loop_ctx.option_sack)
 280                 cookie |= TS_OPT_SACK;
 281         if (tcp_header->ece && tcp_header->cwr)
 282                 cookie |= TS_OPT_ECN;
 283         *tsval = bpf_htonl(cookie);
 284
 285         return true;
 286 }
 287
 288 static __always_inline void values_get_tcpipopts(__u16 *mss, __u8 *wscale,
 289                                                  __u8 *ttl, bool ipv6)
 290 {
 291         __u32 key = 0;
 292         __u64 *value;
 293
 294         value = bpf_map_lookup_elem(&values, &key);
 295         if (value && *value != 0) {
 296                 if (ipv6)
 297                         *mss = (*value >> 32) & 0xffff;
 298                 else
 299                         *mss = *value & 0xffff;
 300                 *wscale = (*value >> 16) & 0xf;
 301                 *ttl = (*value >> 24) & 0xff;
 302                 return;
 303         }
 304
 305         *mss = ipv6 ? DEFAULT_MSS6 : DEFAULT_MSS4;
 306         *wscale = DEFAULT_WSCALE;
 307         *ttl = DEFAULT_TTL;
 308 }
 309
 310 static __always_inline void values_inc_synacks(void)
 311 {
 312         __u32 key = 1;
 313         __u64 *value;
 314
 315         value = bpf_map_lookup_elem(&values, &key);
 316         if (value)
 317                 __sync_fetch_and_add(value, 1);
 318 }
 319
 320 static __always_inline bool check_port_allowed(__u16 port)
 321 {
 322         __u32 i;
 323
 324         for (i = 0; i < MAX_ALLOWED_PORTS; i++) {
 325                 __u32 key = i;
 326                 __u16 *value;
 327
 328                 value = bpf_map_lookup_elem(&allowed_ports, &key);
 329
 330                 if (!value)
 331                         break;
 332                 /* 0 is a terminator value. Check it first to avoid matching on
 333                  * a forbidden port == 0 and returning true.
 334                  */
 335                 if (*value == 0)
 336                         break;
 337
 338                 if (*value == port)
 339                         return true;
 340         }
 341
 342         return false;
 343 }
 344
 345 struct header_pointers {
 346         struct ethhdr *eth;
 347         struct iphdr *ipv4;
 348         struct ipv6hdr *ipv6;
 349         struct tcphdr *tcp;
 350         __u16 tcp_len;
 351 };
 352
 353 static __always_inline int tcp_dissect(void *data, void *data_end,
 354                                        struct header_pointers *hdr)
 355 {
 356         hdr->eth = data;
 357         if (hdr->eth + 1 > data_end)
 358                 return XDP_DROP;
 359
 360         switch (bpf_ntohs(hdr->eth->h_proto)) {
 361         case ETH_P_IP:
 362                 hdr->ipv6 = NULL;
 363
 364                 hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);
 365                 if (hdr->ipv4 + 1 > data_end)
 366                         return XDP_DROP;
 367                 if (hdr->ipv4->ihl * 4 < sizeof(*hdr->ipv4))
 368                         return XDP_DROP;
 369                 if (hdr->ipv4->version != 4)
 370                         return XDP_DROP;
 371
 372                 if (hdr->ipv4->protocol != IPPROTO_TCP)
 373                         return XDP_PASS;
 374
 375                 hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;
 376                 break;
 377         case ETH_P_IPV6:
 378                 hdr->ipv4 = NULL;
 379
 380                 hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);
 381                 if (hdr->ipv6 + 1 > data_end)
 382                         return XDP_DROP;
 383                 if (hdr->ipv6->version != 6)
 384                         return XDP_DROP;
 385
 386                 /* XXX: Extension headers are not supported and could circumvent
 387                  * XDP SYN flood protection.
 388                  */
 389                 if (hdr->ipv6->nexthdr != NEXTHDR_TCP)
 390                         return XDP_PASS;
 391
 392                 hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);
 393                 break;
 394         default:
 395                 /* XXX: VLANs will circumvent XDP SYN flood protection. */
 396                 return XDP_PASS;
 397         }
 398
 399         if (hdr->tcp + 1 > data_end)
 400                 return XDP_DROP;
 401         hdr->tcp_len = hdr->tcp->doff * 4;
 402         if (hdr->tcp_len < sizeof(*hdr->tcp))
 403                 return XDP_DROP;
 404
 405         return XDP_TX;
 406 }
 407
 408 static __always_inline int tcp_lookup(void *ctx, struct header_pointers *hdr, bool xdp)
 409 {
 410         struct bpf_ct_opts___local ct_lookup_opts = {
 411                 .netns_id = BPF_F_CURRENT_NETNS,
 412                 .l4proto = IPPROTO_TCP,
 413         };
 414         struct bpf_sock_tuple tup = {};
 415         struct nf_conn *ct;
 416         __u32 tup_size;
 417
 418         if (hdr->ipv4) {
 419                 /* TCP doesn't normally use fragments, and XDP can't reassemble
 420                  * them.
 421                  */
 422                 if ((hdr->ipv4->frag_off & bpf_htons(IP_DF | IP_MF | IP_OFFSET)) != bpf_htons(IP_DF))
 423                         return XDP_DROP;
 424
 425                 tup.ipv4.saddr = hdr->ipv4->saddr;
 426                 tup.ipv4.daddr = hdr->ipv4->daddr;
 427                 tup.ipv4.sport = hdr->tcp->source;
 428                 tup.ipv4.dport = hdr->tcp->dest;
 429                 tup_size = sizeof(tup.ipv4);
 430         } else if (hdr->ipv6) {
 431                 __builtin_memcpy(tup.ipv6.saddr, &hdr->ipv6->saddr, sizeof(tup.ipv6.saddr));
 432                 __builtin_memcpy(tup.ipv6.daddr, &hdr->ipv6->daddr, sizeof(tup.ipv6.daddr));
 433                 tup.ipv6.sport = hdr->tcp->source;
 434                 tup.ipv6.dport = hdr->tcp->dest;
 435                 tup_size = sizeof(tup.ipv6);
 436         } else {
 437                 /* The verifier can't track that either ipv4 or ipv6 is not
 438                  * NULL.
 439                  */
 440                 return XDP_ABORTED;
 441         }
 442         if (xdp)
 443                 ct = bpf_xdp_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
 444         else
 445                 ct = bpf_skb_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
 446         if (ct) {
 447                 unsigned long status = ct->status;
 448
 449                 bpf_ct_release(ct);
 450                 if (status & IPS_CONFIRMED)
 451                         return XDP_PASS;
 452         } else if (ct_lookup_opts.error != -ENOENT) {
 453                 return XDP_ABORTED;
 454         }
 455
 456         /* error == -ENOENT || !(status & IPS_CONFIRMED) */
 457         return XDP_TX;
 458 }
 459
 460 static __always_inline __u8 tcp_mkoptions(__be32 *buf, __be32 *tsopt, __u16 mss,
 461                                           __u8 wscale)
 462 {
 463         __be32 *start = buf;
 464
 465         *buf++ = bpf_htonl((TCPOPT_MSS << 24) | (TCPOLEN_MSS << 16) | mss);
 466
 467         if (!tsopt)
 468                 return buf - start;
 469
 470         if (tsopt[0] & bpf_htonl(1 << 4))
 471                 *buf++ = bpf_htonl((TCPOPT_SACK_PERM << 24) |
 472                                    (TCPOLEN_SACK_PERM << 16) |
 473                                    (TCPOPT_TIMESTAMP << 8) |
 474                                    TCPOLEN_TIMESTAMP);
 475         else
 476                 *buf++ = bpf_htonl((TCPOPT_NOP << 24) |
 477                                    (TCPOPT_NOP << 16) |
 478                                    (TCPOPT_TIMESTAMP << 8) |
 479                                    TCPOLEN_TIMESTAMP);
 480         *buf++ = tsopt[0];
 481         *buf++ = tsopt[1];
 482
 483         if ((tsopt[0] & bpf_htonl(0xf)) != bpf_htonl(0xf))
 484                 *buf++ = bpf_htonl((TCPOPT_NOP << 24) |
 485                                    (TCPOPT_WINDOW << 16) |
 486                                    (TCPOLEN_WINDOW << 8) |
 487                                    wscale);
 488
 489         return buf - start;
 490 }
 491
 492 static __always_inline void tcp_gen_synack(struct tcphdr *tcp_header,
 493                                            __u32 cookie, __be32 *tsopt,
 494                                            __u16 mss, __u8 wscale)
 495 {
 496         void *tcp_options;
 497
 498         tcp_flag_word(tcp_header) = TCP_FLAG_SYN | TCP_FLAG_ACK;
 499         if (tsopt && (tsopt[0] & bpf_htonl(1 << 5)))
 500                 tcp_flag_word(tcp_header) |= TCP_FLAG_ECE;
 501         tcp_header->doff = 5; /* doff is part of tcp_flag_word. */
 502         swap(tcp_header->source, tcp_header->dest);
 503         tcp_header->ack_seq = bpf_htonl(bpf_ntohl(tcp_header->seq) + 1);
 504         tcp_header->seq = bpf_htonl(cookie);
 505         tcp_header->window = 0;
 506         tcp_header->urg_ptr = 0;
 507         tcp_header->check = 0; /* Calculate checksum later. */
 508
 509         tcp_options = (void *)(tcp_header + 1);
 510         tcp_header->doff += tcp_mkoptions(tcp_options, tsopt, mss, wscale);
 511 }
 512
 513 static __always_inline void tcpv4_gen_synack(struct header_pointers *hdr,
 514                                              __u32 cookie, __be32 *tsopt)
 515 {
 516         __u8 wscale;
 517         __u16 mss;
 518         __u8 ttl;
 519
 520         values_get_tcpipopts(&mss, &wscale, &ttl, false);
 521
 522         swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
 523
 524         swap(hdr->ipv4->saddr, hdr->ipv4->daddr);
 525         hdr->ipv4->check = 0; /* Calculate checksum later. */
 526         hdr->ipv4->tos = 0;
 527         hdr->ipv4->id = 0;
 528         hdr->ipv4->ttl = ttl;
 529
 530         tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
 531
 532         hdr->tcp_len = hdr->tcp->doff * 4;
 533         hdr->ipv4->tot_len = bpf_htons(sizeof(*hdr->ipv4) + hdr->tcp_len);
 534 }
 535
 536 static __always_inline void tcpv6_gen_synack(struct header_pointers *hdr,
 537                                              __u32 cookie, __be32 *tsopt)
 538 {
 539         __u8 wscale;
 540         __u16 mss;
 541         __u8 ttl;
 542
 543         values_get_tcpipopts(&mss, &wscale, &ttl, true);
 544
 545         swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
 546
 547         swap(hdr->ipv6->saddr, hdr->ipv6->daddr);
 548         *(__be32 *)hdr->ipv6 = bpf_htonl(0x60000000);
 549         hdr->ipv6->hop_limit = ttl;
 550
 551         tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
 552
 553         hdr->tcp_len = hdr->tcp->doff * 4;
 554         hdr->ipv6->payload_len = bpf_htons(hdr->tcp_len);
 555 }
 556
 557 static __always_inline int syncookie_handle_syn(struct header_pointers *hdr,
 558                                                 void *ctx,
 559                                                 void *data, void *data_end,
 560                                                 bool xdp)
 561 {
 562         __u32 old_pkt_size, new_pkt_size;
 563         /* Unlike clang 10, clang 11 and 12 generate code that doesn't pass the
 564          * BPF verifier if tsopt is not volatile. Volatile forces it to store
 565          * the pointer value and use it directly, otherwise tcp_mkoptions is
 566          * (mis)compiled like this:
 567          *   if (!tsopt)
 568          *       return buf - start;
 569          *   reg = stored_return_value_of_tscookie_init;
 570          *   if (reg)
 571          *       tsopt = tsopt_buf;
 572          *   else
 573          *       tsopt = NULL;
 574          *   ...
 575          *   *buf++ = tsopt[1];
 576          * It creates a dead branch where tsopt is assigned NULL, but the
 577          * verifier can't prove it's dead and blocks the program.
 578          */
 579         __be32 * volatile tsopt = NULL;
 580         __be32 tsopt_buf[2] = {};
 581         __u16 ip_len;
 582         __u32 cookie;
 583         __s64 value;
 584
 585         /* Checksum is not yet verified, but both checksum failure and TCP
 586          * header checks return XDP_DROP, so the order doesn't matter.
 587          */
 588         if (hdr->tcp->fin || hdr->tcp->rst)
 589                 return XDP_DROP;
 590
 591         /* Issue SYN cookies on allowed ports, drop SYN packets on blocked
 592          * ports.
 593          */
 594         if (!check_port_allowed(bpf_ntohs(hdr->tcp->dest)))
 595                 return XDP_DROP;
 596
 597         if (hdr->ipv4) {
 598                 /* Check the IPv4 and TCP checksums before creating a SYNACK. */
 599                 value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, hdr->ipv4->ihl * 4, 0);
 600                 if (value < 0)
 601                         return XDP_ABORTED;
 602                 if (csum_fold(value) != 0)
 603                         return XDP_DROP; /* Bad IPv4 checksum. */
 604
 605                 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
 606                 if (value < 0)
 607                         return XDP_ABORTED;
 608                 if (csum_tcpudp_magic(hdr->ipv4->saddr, hdr->ipv4->daddr,
 609                                       hdr->tcp_len, IPPROTO_TCP, value) != 0)
 610                         return XDP_DROP; /* Bad TCP checksum. */
 611
 612                 ip_len = sizeof(*hdr->ipv4);
 613
 614                 value = bpf_tcp_raw_gen_syncookie_ipv4(hdr->ipv4, hdr->tcp,
 615                                                        hdr->tcp_len);
 616         } else if (hdr->ipv6) {
 617                 /* Check the TCP checksum before creating a SYNACK. */
 618                 value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
 619                 if (value < 0)
 620                         return XDP_ABORTED;
 621                 if (csum_ipv6_magic(&hdr->ipv6->saddr, &hdr->ipv6->daddr,
 622                                     hdr->tcp_len, IPPROTO_TCP, value) != 0)
 623                         return XDP_DROP; /* Bad TCP checksum. */
 624
 625                 ip_len = sizeof(*hdr->ipv6);
 626
 627                 value = bpf_tcp_raw_gen_syncookie_ipv6(hdr->ipv6, hdr->tcp,
 628                                                        hdr->tcp_len);
 629         } else {
 630                 return XDP_ABORTED;
 631         }
 632
 633         if (value < 0)
 634                 return XDP_ABORTED;
 635         cookie = (__u32)value;
 636
 637         if (tscookie_init((void *)hdr->tcp, hdr->tcp_len,
 638                           &tsopt_buf[0], &tsopt_buf[1], data_end))
 639                 tsopt = tsopt_buf;
 640
 641         /* Check that there is enough space for a SYNACK. It also covers
 642          * the check that the destination of the __builtin_memmove below
 643          * doesn't overflow.
 644          */
 645         if (data + sizeof(*hdr->eth) + ip_len + TCP_MAXLEN > data_end)
 646                 return XDP_ABORTED;
 647
 648         if (hdr->ipv4) {
 649                 if (hdr->ipv4->ihl * 4 > sizeof(*hdr->ipv4)) {
 650                         struct tcphdr *new_tcp_header;
 651
 652                         new_tcp_header = data + sizeof(*hdr->eth) + sizeof(*hdr->ipv4);
 653                         __builtin_memmove(new_tcp_header, hdr->tcp, sizeof(*hdr->tcp));
 654                         hdr->tcp = new_tcp_header;
 655
 656                         hdr->ipv4->ihl = sizeof(*hdr->ipv4) / 4;
 657                 }
 658
 659                 tcpv4_gen_synack(hdr, cookie, tsopt);
 660         } else if (hdr->ipv6) {
 661                 tcpv6_gen_synack(hdr, cookie, tsopt);
 662         } else {
 663                 return XDP_ABORTED;
 664         }
 665
 666         /* Recalculate checksums. */
 667         hdr->tcp->check = 0;
 668         value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
 669         if (value < 0)
 670                 return XDP_ABORTED;
 671         if (hdr->ipv4) {
 672                 hdr->tcp->check = csum_tcpudp_magic(hdr->ipv4->saddr,
 673                                                     hdr->ipv4->daddr,
 674                                                     hdr->tcp_len,
 675                                                     IPPROTO_TCP,
 676                                                     value);
 677
 678                 hdr->ipv4->check = 0;
 679                 value = bpf_csum_diff(0, 0, (void *)hdr->ipv4, sizeof(*hdr->ipv4), 0);
 680                 if (value < 0)
 681                         return XDP_ABORTED;
 682                 hdr->ipv4->check = csum_fold(value);
 683         } else if (hdr->ipv6) {
 684                 hdr->tcp->check = csum_ipv6_magic(&hdr->ipv6->saddr,
 685                                                   &hdr->ipv6->daddr,
 686                                                   hdr->tcp_len,
 687                                                   IPPROTO_TCP,
 688                                                   value);
 689         } else {
 690                 return XDP_ABORTED;
 691         }
 692
 693         /* Set the new packet size. */
 694         old_pkt_size = data_end - data;
 695         new_pkt_size = sizeof(*hdr->eth) + ip_len + hdr->tcp->doff * 4;
 696         if (xdp) {
 697                 if (bpf_xdp_adjust_tail(ctx, new_pkt_size - old_pkt_size))
 698                         return XDP_ABORTED;
 699         } else {
 700                 if (bpf_skb_change_tail(ctx, new_pkt_size, 0))
 701                         return XDP_ABORTED;
 702         }
 703
 704         values_inc_synacks();
 705
 706         return XDP_TX;
 707 }
 708
 709 static __always_inline int syncookie_handle_ack(struct header_pointers *hdr)
 710 {
 711         int err;
 712
 713         if (hdr->tcp->rst)
 714                 return XDP_DROP;
 715
 716         if (hdr->ipv4)
 717                 err = bpf_tcp_raw_check_syncookie_ipv4(hdr->ipv4, hdr->tcp);
 718         else if (hdr->ipv6)
 719                 err = bpf_tcp_raw_check_syncookie_ipv6(hdr->ipv6, hdr->tcp);
 720         else
 721                 return XDP_ABORTED;
 722         if (err)
 723                 return XDP_DROP;
 724
 725         return XDP_PASS;
 726 }
 727
 728 static __always_inline int syncookie_part1(void *ctx, void *data, void *data_end,
 729                                            struct header_pointers *hdr, bool xdp)
 730 {
 731         int ret;
 732
 733         ret = tcp_dissect(data, data_end, hdr);
 734         if (ret != XDP_TX)
 735                 return ret;
 736
 737         ret = tcp_lookup(ctx, hdr, xdp);
 738         if (ret != XDP_TX)
 739                 return ret;
 740
 741         /* Packet is TCP and doesn't belong to an established connection. */
 742
 743         if ((hdr->tcp->syn ^ hdr->tcp->ack) != 1)
 744                 return XDP_DROP;
 745
 746         /* Grow the TCP header to TCP_MAXLEN to be able to pass any hdr->tcp_len
 747          * to bpf_tcp_raw_gen_syncookie_ipv{4,6} and pass the verifier.
 748          */
 749         if (xdp) {
 750                 if (bpf_xdp_adjust_tail(ctx, TCP_MAXLEN - hdr->tcp_len))
 751                         return XDP_ABORTED;
 752         } else {
 753                 /* Without volatile the verifier throws this error:
 754                  * R9 32-bit pointer arithmetic prohibited
 755                  */
 756                 volatile u64 old_len = data_end - data;
 757
 758                 if (bpf_skb_change_tail(ctx, old_len + TCP_MAXLEN - hdr->tcp_len, 0))
 759                         return XDP_ABORTED;
 760         }
 761
 762         return XDP_TX;
 763 }
 764
 765 static __always_inline int syncookie_part2(void *ctx, void *data, void *data_end,
 766                                            struct header_pointers *hdr, bool xdp)
 767 {
 768         if (hdr->ipv4) {
 769                 hdr->eth = data;
 770                 hdr->ipv4 = (void *)hdr->eth + sizeof(*hdr->eth);
 771                 /* IPV4_MAXLEN is needed when calculating checksum.
 772                  * At least sizeof(struct iphdr) is needed here to access ihl.
 773                  */
 774                 if ((void *)hdr->ipv4 + IPV4_MAXLEN > data_end)
 775                         return XDP_ABORTED;
 776                 hdr->tcp = (void *)hdr->ipv4 + hdr->ipv4->ihl * 4;
 777         } else if (hdr->ipv6) {
 778                 hdr->eth = data;
 779                 hdr->ipv6 = (void *)hdr->eth + sizeof(*hdr->eth);
 780                 hdr->tcp = (void *)hdr->ipv6 + sizeof(*hdr->ipv6);
 781         } else {
 782                 return XDP_ABORTED;
 783         }
 784
 785         if ((void *)hdr->tcp + TCP_MAXLEN > data_end)
 786                 return XDP_ABORTED;
 787
 788         /* We run out of registers, tcp_len gets spilled to the stack, and the
 789          * verifier forgets its min and max values checked above in tcp_dissect.
 790          */
 791         hdr->tcp_len = hdr->tcp->doff * 4;
 792         if (hdr->tcp_len < sizeof(*hdr->tcp))
 793                 return XDP_ABORTED;
 794
 795         return hdr->tcp->syn ? syncookie_handle_syn(hdr, ctx, data, data_end, xdp) :
 796                                syncookie_handle_ack(hdr);
 797 }
 798
 799 SEC("xdp")
 800 int syncookie_xdp(struct xdp_md *ctx)
 801 {
 802         void *data_end = (void *)(long)ctx->data_end;
 803         void *data = (void *)(long)ctx->data;
 804         struct header_pointers hdr;
 805         int ret;
 806
 807         ret = syncookie_part1(ctx, data, data_end, &hdr, true);
 808         if (ret != XDP_TX)
 809                 return ret;
 810
 811         data_end = (void *)(long)ctx->data_end;
 812         data = (void *)(long)ctx->data;
 813
 814         return syncookie_part2(ctx, data, data_end, &hdr, true);
 815 }
 816
 817 SEC("tc")
 818 int syncookie_tc(struct __sk_buff *skb)
 819 {
 820         void *data_end = (void *)(long)skb->data_end;
 821         void *data = (void *)(long)skb->data;
 822         struct header_pointers hdr;
 823         int ret;
 824
 825         ret = syncookie_part1(skb, data, data_end, &hdr, false);
 826         if (ret != XDP_TX)
 827                 return ret == XDP_PASS ? TC_ACT_OK : TC_ACT_SHOT;
 828
 829         data_end = (void *)(long)skb->data_end;
 830         data = (void *)(long)skb->data;
 831
 832         ret = syncookie_part2(skb, data, data_end, &hdr, false);
 833         switch (ret) {
 834         case XDP_PASS:
 835                 return TC_ACT_OK;
 836         case XDP_TX:
 837                 return bpf_redirect(skb->ifindex, 0);
 838         default:
 839                 return TC_ACT_SHOT;
 840         }
 841 }
 842
 843 char _license[] SEC("license") = "GPL";