1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2020 Intel Corporation. */
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
57 * - Single process spawns two threads: Tx and Rx
58 * - Each of these two threads attach to a veth interface
59 * - Each thread creates one AF_XDP socket connected to a unique umem for each
61 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
62 * - Rx thread verifies if all packets were received and delivered in-order,
63 * and have the right content
65 * Enable/disable packet dump mode:
66 * --------------------------
67 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
68 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
75 #include <asm/barrier.h>
76 #include <linux/if_link.h>
77 #include <linux/if_ether.h>
79 #include <linux/udp.h>
80 #include <arpa/inet.h>
92 #include <sys/socket.h>
94 #include <sys/types.h>
95 #include <sys/queue.h>
98 #include <stdatomic.h>
100 #include "xsk_xdp_progs.skel.h"
102 #include "xskxceiver.h"
104 #include <linux/filter.h>
105 #include "../kselftest.h"
106 #include "xsk_xdp_metadata.h"
108 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
109 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
110 static const char *IP1 = "192.168.100.162";
111 static const char *IP2 = "192.168.100.161";
112 static const u16 UDP_PORT1 = 2020;
113 static const u16 UDP_PORT2 = 2121;
115 static void __exit_with_error(int error, const char *file, const char *func, int line)
117 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
122 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
123 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
124 static char *mode_string(struct test_spec *test)
126 switch (test->mode) {
138 static void report_failure(struct test_spec *test)
143 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
148 static void memset32_htonl(void *dest, u32 val, u32 size)
150 u32 *ptr = (u32 *)dest;
155 for (i = 0; i < (size & (~0x3)); i += 4)
160 * Fold a partial checksum
161 * This function code has been taken from
162 * Linux kernel include/asm-generic/checksum.h
164 static __u16 csum_fold(__u32 csum)
166 u32 sum = (__force u32)csum;
168 sum = (sum & 0xffff) + (sum >> 16);
169 sum = (sum & 0xffff) + (sum >> 16);
170 return (__force __u16)~sum;
174 * This function code has been taken from
175 * Linux kernel lib/checksum.c
177 static u32 from64to32(u64 x)
179 /* add up 32-bit and 32-bit for 32+c bit */
180 x = (x & 0xffffffff) + (x >> 32);
182 x = (x & 0xffffffff) + (x >> 32);
187 * This function code has been taken from
188 * Linux kernel lib/checksum.c
190 static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
192 unsigned long long s = (__force u32)sum;
194 s += (__force u32)saddr;
195 s += (__force u32)daddr;
196 #ifdef __BIG_ENDIAN__
199 s += (proto + len) << 8;
201 return (__force __u32)from64to32(s);
205 * This function has been taken from
206 * Linux kernel include/asm-generic/checksum.h
208 static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
210 return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
213 static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
218 /* udp hdr and data */
219 for (; cnt < len; cnt += 2)
220 csum += udp_pkt[cnt >> 1];
222 return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
225 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
227 memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
228 memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
229 eth_hdr->h_proto = htons(ETH_P_IP);
232 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)
234 ip_hdr->version = IP_PKT_VER;
236 ip_hdr->tos = IP_PKT_TOS;
237 ip_hdr->tot_len = htons(IP_PKT_SIZE);
239 ip_hdr->frag_off = 0;
240 ip_hdr->ttl = IPDEFTTL;
241 ip_hdr->protocol = IPPROTO_UDP;
242 ip_hdr->saddr = ifobject->src_ip;
243 ip_hdr->daddr = ifobject->dst_ip;
247 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject,
248 struct udphdr *udp_hdr)
250 udp_hdr->source = htons(ifobject->src_port);
251 udp_hdr->dest = htons(ifobject->dst_port);
252 udp_hdr->len = htons(UDP_PKT_SIZE);
253 memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE);
256 static bool is_umem_valid(struct ifobject *ifobj)
258 return !!ifobj->umem->umem;
261 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
265 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);
268 static u32 mode_to_xdp_flags(enum test_mode mode)
270 return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
273 static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size)
275 struct xsk_umem_config cfg = {
276 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
277 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
278 .frame_size = umem->frame_size,
279 .frame_headroom = umem->frame_headroom,
280 .flags = XSK_UMEM__DEFAULT_FLAGS
284 if (umem->unaligned_mode)
285 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
287 ret = xsk_umem__create(&umem->umem, buffer, size,
288 &umem->fq, &umem->cq, &cfg);
292 umem->buffer = buffer;
296 static void enable_busy_poll(struct xsk_socket_info *xsk)
301 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
302 (void *)&sock_opt, sizeof(sock_opt)) < 0)
303 exit_with_error(errno);
306 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
307 (void *)&sock_opt, sizeof(sock_opt)) < 0)
308 exit_with_error(errno);
310 sock_opt = BATCH_SIZE;
311 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
312 (void *)&sock_opt, sizeof(sock_opt)) < 0)
313 exit_with_error(errno);
316 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
317 struct ifobject *ifobject, bool shared)
319 struct xsk_socket_config cfg = {};
320 struct xsk_ring_cons *rxr;
321 struct xsk_ring_prod *txr;
324 cfg.rx_size = xsk->rxqsize;
325 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
326 cfg.bind_flags = ifobject->bind_flags;
328 cfg.bind_flags |= XDP_SHARED_UMEM;
330 txr = ifobject->tx_on ? &xsk->tx : NULL;
331 rxr = ifobject->rx_on ? &xsk->rx : NULL;
332 return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);
335 static bool ifobj_zc_avail(struct ifobject *ifobject)
337 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
338 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
339 struct xsk_socket_info *xsk;
340 struct xsk_umem_info *umem;
341 bool zc_avail = false;
345 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
346 if (bufs == MAP_FAILED)
347 exit_with_error(errno);
349 umem = calloc(1, sizeof(struct xsk_umem_info));
351 munmap(bufs, umem_sz);
352 exit_with_error(ENOMEM);
354 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
355 ret = xsk_configure_umem(umem, bufs, umem_sz);
357 exit_with_error(-ret);
359 xsk = calloc(1, sizeof(struct xsk_socket_info));
362 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
363 ifobject->rx_on = true;
364 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
365 ret = __xsk_configure_socket(xsk, umem, ifobject, false);
369 xsk_socket__delete(xsk->xsk);
372 munmap(umem->buffer, umem_sz);
373 xsk_umem__delete(umem->umem);
378 static struct option long_options[] = {
379 {"interface", required_argument, 0, 'i'},
380 {"busy-poll", no_argument, 0, 'b'},
381 {"dump-pkts", no_argument, 0, 'D'},
382 {"verbose", no_argument, 0, 'v'},
386 static void usage(const char *prog)
389 " Usage: %s [OPTIONS]\n"
391 " -i, --interface Use interface\n"
392 " -D, --dump-pkts Dump packets L2 - L5\n"
393 " -v, --verbose Verbose output\n"
394 " -b, --busy-poll Enable busy poll\n";
396 ksft_print_msg(str, prog);
399 static bool validate_interface(struct ifobject *ifobj)
401 if (!strcmp(ifobj->ifname, ""))
406 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
409 struct ifobject *ifobj;
410 u32 interface_nb = 0;
416 c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index);
422 if (interface_nb == 0)
424 else if (interface_nb == 1)
429 memcpy(ifobj->ifname, optarg,
430 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
432 ifobj->ifindex = if_nametoindex(ifobj->ifname);
434 exit_with_error(errno);
445 ifobj_tx->busy_poll = true;
446 ifobj_rx->busy_poll = true;
449 usage(basename(argv[0]));
455 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
456 struct ifobject *ifobj_rx)
460 for (i = 0; i < MAX_INTERFACES; i++) {
461 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
463 ifobj->xsk = &ifobj->xsk_arr[0];
464 ifobj->use_poll = false;
465 ifobj->use_fill_ring = true;
466 ifobj->release_rx = true;
467 ifobj->validation_func = NULL;
468 ifobj->use_metadata = false;
471 ifobj->rx_on = false;
473 ifobj->pkt_stream = test->tx_pkt_stream_default;
476 ifobj->tx_on = false;
477 ifobj->pkt_stream = test->rx_pkt_stream_default;
480 memset(ifobj->umem, 0, sizeof(*ifobj->umem));
481 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
482 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
483 if (ifobj->shared_umem && ifobj->rx_on)
484 ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS *
485 XSK_UMEM__DEFAULT_FRAME_SIZE;
487 for (j = 0; j < MAX_SOCKETS; j++) {
488 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
489 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
493 test->ifobj_tx = ifobj_tx;
494 test->ifobj_rx = ifobj_rx;
495 test->current_step = 0;
496 test->total_steps = 1;
497 test->nb_sockets = 1;
499 test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog;
500 test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk;
501 test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;
502 test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;
505 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
506 struct ifobject *ifobj_rx, enum test_mode mode)
508 struct pkt_stream *tx_pkt_stream;
509 struct pkt_stream *rx_pkt_stream;
512 tx_pkt_stream = test->tx_pkt_stream_default;
513 rx_pkt_stream = test->rx_pkt_stream_default;
514 memset(test, 0, sizeof(*test));
515 test->tx_pkt_stream_default = tx_pkt_stream;
516 test->rx_pkt_stream_default = rx_pkt_stream;
518 for (i = 0; i < MAX_INTERFACES; i++) {
519 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
521 ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
522 if (mode == TEST_MODE_ZC)
523 ifobj->bind_flags |= XDP_ZEROCOPY;
525 ifobj->bind_flags |= XDP_COPY;
529 __test_spec_init(test, ifobj_tx, ifobj_rx);
532 static void test_spec_reset(struct test_spec *test)
534 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
537 static void test_spec_set_name(struct test_spec *test, const char *name)
539 strncpy(test->name, name, MAX_TEST_NAME_SIZE);
542 static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx,
543 struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx,
544 struct bpf_map *xskmap_tx)
546 test->xdp_prog_rx = xdp_prog_rx;
547 test->xdp_prog_tx = xdp_prog_tx;
548 test->xskmap_rx = xskmap_rx;
549 test->xskmap_tx = xskmap_tx;
552 static void pkt_stream_reset(struct pkt_stream *pkt_stream)
555 pkt_stream->rx_pkt_nb = 0;
558 static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb)
560 if (pkt_nb >= pkt_stream->nb_pkts)
563 return &pkt_stream->pkts[pkt_nb];
566 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
568 while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) {
570 if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid)
571 return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++];
572 pkt_stream->rx_pkt_nb++;
577 static void pkt_stream_delete(struct pkt_stream *pkt_stream)
579 free(pkt_stream->pkts);
583 static void pkt_stream_restore_default(struct test_spec *test)
585 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
586 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
588 if (tx_pkt_stream != test->tx_pkt_stream_default) {
589 pkt_stream_delete(test->ifobj_tx->pkt_stream);
590 test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
593 if (rx_pkt_stream != test->rx_pkt_stream_default) {
594 pkt_stream_delete(test->ifobj_rx->pkt_stream);
595 test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
599 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
601 struct pkt_stream *pkt_stream;
603 pkt_stream = calloc(1, sizeof(*pkt_stream));
607 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
608 if (!pkt_stream->pkts) {
613 pkt_stream->nb_pkts = nb_pkts;
617 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len)
619 pkt->addr = addr + umem->base_addr;
621 if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)
627 static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
629 struct pkt_stream *pkt_stream;
632 pkt_stream = __pkt_stream_alloc(nb_pkts);
634 exit_with_error(ENOMEM);
636 pkt_stream->nb_pkts = nb_pkts;
637 for (i = 0; i < nb_pkts; i++) {
638 pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size,
640 pkt_stream->pkts[i].payload = i;
646 static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
647 struct pkt_stream *pkt_stream)
649 return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
652 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
654 struct pkt_stream *pkt_stream;
656 pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
657 test->ifobj_tx->pkt_stream = pkt_stream;
658 pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
659 test->ifobj_rx->pkt_stream = pkt_stream;
662 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
665 struct xsk_umem_info *umem = ifobj->umem;
666 struct pkt_stream *pkt_stream;
669 pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
670 for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
671 pkt_set(umem, &pkt_stream->pkts[i],
672 (i % umem->num_frames) * umem->frame_size + offset, pkt_len);
674 ifobj->pkt_stream = pkt_stream;
677 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
679 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
680 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
683 static void pkt_stream_receive_half(struct test_spec *test)
685 struct xsk_umem_info *umem = test->ifobj_rx->umem;
686 struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream;
689 test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts,
690 pkt_stream->pkts[0].len);
691 pkt_stream = test->ifobj_rx->pkt_stream;
692 for (i = 1; i < pkt_stream->nb_pkts; i += 2)
693 pkt_stream->pkts[i].valid = false;
696 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)
698 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);
699 struct udphdr *udp_hdr;
700 struct ethhdr *eth_hdr;
701 struct iphdr *ip_hdr;
706 if (!pkt->valid || pkt->len < MIN_PKT_SIZE)
709 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);
710 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr));
711 ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr));
712 eth_hdr = (struct ethhdr *)data;
714 gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr);
715 gen_ip_hdr(ifobject, ip_hdr);
716 gen_udp_csum(udp_hdr, ip_hdr);
717 gen_eth_hdr(ifobject, eth_hdr);
722 static void __pkt_stream_generate_custom(struct ifobject *ifobj,
723 struct pkt *pkts, u32 nb_pkts)
725 struct pkt_stream *pkt_stream;
728 pkt_stream = __pkt_stream_alloc(nb_pkts);
730 exit_with_error(ENOMEM);
732 for (i = 0; i < nb_pkts; i++) {
733 pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr;
734 pkt_stream->pkts[i].len = pkts[i].len;
735 pkt_stream->pkts[i].payload = i;
736 pkt_stream->pkts[i].valid = pkts[i].valid;
739 ifobj->pkt_stream = pkt_stream;
742 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
744 __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts);
745 __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts);
748 static void pkt_dump(void *pkt, u32 len)
750 char s[INET_ADDRSTRLEN];
751 struct ethhdr *ethhdr;
752 struct udphdr *udphdr;
757 iphdr = pkt + sizeof(*ethhdr);
758 udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr);
760 /*extract L2 frame */
761 fprintf(stdout, "DEBUG>> L2: dst mac: ");
762 for (i = 0; i < ETH_ALEN; i++)
763 fprintf(stdout, "%02X", ethhdr->h_dest[i]);
765 fprintf(stdout, "\nDEBUG>> L2: src mac: ");
766 for (i = 0; i < ETH_ALEN; i++)
767 fprintf(stdout, "%02X", ethhdr->h_source[i]);
769 /*extract L3 frame */
770 fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);
771 fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",
772 inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));
773 fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",
774 inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));
775 /*extract L4 frame */
776 fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));
777 fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));
778 /*extract L5 frame */
779 payload = ntohl(*((u32 *)(pkt + PKT_HDR_SIZE)));
781 fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload);
782 fprintf(stdout, "---------------------------------------\n");
785 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr,
788 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
789 u32 offset = addr % umem->frame_size, expected_offset = 0;
791 if (!pkt_stream->use_addr_for_fill)
794 expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
796 if (offset == expected_offset)
799 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
803 static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr)
805 void *data = xsk_umem__get_data(buffer, addr);
806 struct xdp_info *meta = data - sizeof(struct xdp_info);
808 if (meta->count != pkt->payload) {
809 ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%d]\n",
810 __func__, pkt->payload, meta->count);
817 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
819 void *data = xsk_umem__get_data(buffer, addr);
820 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr));
823 ksft_print_msg("[%s] too many packets received\n", __func__);
827 if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) {
828 /* Do not try to verify packets that are smaller than minimum size. */
832 if (pkt->len != len) {
833 ksft_print_msg("[%s] expected length [%d], got length [%d]\n",
834 __func__, pkt->len, len);
838 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
839 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE)));
842 pkt_dump(data, PKT_SIZE);
844 if (pkt->payload != seqnum) {
845 ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n",
846 __func__, pkt->payload, seqnum);
850 ksft_print_msg("Invalid frame received: ");
851 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,
859 static void kick_tx(struct xsk_socket_info *xsk)
863 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
866 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
870 exit_with_error(errno);
873 static void kick_rx(struct xsk_socket_info *xsk)
877 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
879 exit_with_error(errno);
882 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
887 if (xsk_ring_prod__needs_wakeup(&xsk->tx))
890 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
892 if (rcvd > xsk->outstanding_tx) {
893 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
895 ksft_print_msg("[%s] Too many packets completed\n", __func__);
896 ksft_print_msg("Last completion address: %llx\n", addr);
900 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
901 xsk->outstanding_tx -= rcvd;
907 static int receive_pkts(struct test_spec *test, struct pollfd *fds)
909 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
910 struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream;
911 u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0;
912 struct xsk_socket_info *xsk = test->ifobj_rx->xsk;
913 struct ifobject *ifobj = test->ifobj_rx;
914 struct xsk_umem_info *umem = xsk->umem;
918 ret = gettimeofday(&tv_now, NULL);
920 exit_with_error(errno);
921 timeradd(&tv_now, &tv_timeout, &tv_end);
923 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
925 ret = gettimeofday(&tv_now, NULL);
927 exit_with_error(errno);
928 if (timercmp(&tv_now, &tv_end, >)) {
929 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
934 if (ifobj->use_poll) {
935 ret = poll(fds, 1, POLL_TMOUT);
937 exit_with_error(errno);
940 if (!is_umem_valid(test->ifobj_tx))
943 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
948 if (!(fds->revents & POLLIN))
952 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
956 if (ifobj->use_fill_ring) {
957 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
958 while (ret != rcvd) {
960 exit_with_error(-ret);
961 if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
962 ret = poll(fds, 1, POLL_TMOUT);
964 exit_with_error(errno);
966 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
970 for (i = 0; i < rcvd; i++) {
971 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
972 u64 addr = desc->addr, orig;
974 orig = xsk_umem__extract_addr(addr);
975 addr = xsk_umem__add_offset_to_addr(addr);
977 if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) ||
978 !is_offset_correct(umem, pkt_stream, addr, pkt->addr) ||
979 (ifobj->use_metadata && !is_metadata_correct(pkt, umem->buffer, addr)))
982 if (ifobj->use_fill_ring)
983 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
984 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
987 if (ifobj->use_fill_ring)
988 xsk_ring_prod__submit(&umem->fq, rcvd);
989 if (ifobj->release_rx)
990 xsk_ring_cons__release(&xsk->rx, rcvd);
992 pthread_mutex_lock(&pacing_mutex);
993 pkts_in_flight -= pkts_sent;
994 if (pkts_in_flight < umem->num_frames)
995 pthread_cond_signal(&pacing_cond);
996 pthread_mutex_unlock(&pacing_mutex);
1003 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds,
1006 struct xsk_socket_info *xsk = ifobject->xsk;
1007 bool use_poll = ifobject->use_poll;
1008 u32 i, idx = 0, valid_pkts = 0;
1011 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
1013 ret = poll(fds, 1, POLL_TMOUT);
1016 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1018 return TEST_FAILURE;
1025 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1027 return TEST_FAILURE;
1031 complete_pkts(xsk, BATCH_SIZE);
1034 for (i = 0; i < BATCH_SIZE; i++) {
1035 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
1036 struct pkt *pkt = pkt_generate(ifobject, *pkt_nb);
1041 tx_desc->addr = pkt->addr;
1042 tx_desc->len = pkt->len;
1048 pthread_mutex_lock(&pacing_mutex);
1049 pkts_in_flight += valid_pkts;
1050 /* pkts_in_flight might be negative if many invalid packets are sent */
1051 if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) {
1053 pthread_cond_wait(&pacing_cond, &pacing_mutex);
1055 pthread_mutex_unlock(&pacing_mutex);
1057 xsk_ring_prod__submit(&xsk->tx, i);
1058 xsk->outstanding_tx += valid_pkts;
1061 ret = poll(fds, 1, POLL_TMOUT);
1063 if (ret == 0 && timeout)
1066 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1067 return TEST_FAILURE;
1072 if (complete_pkts(xsk, i))
1073 return TEST_FAILURE;
1079 return TEST_CONTINUE;
1082 static void wait_for_tx_completion(struct xsk_socket_info *xsk)
1084 while (xsk->outstanding_tx)
1085 complete_pkts(xsk, BATCH_SIZE);
1088 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1090 bool timeout = !is_umem_valid(test->ifobj_rx);
1091 struct pollfd fds = { };
1092 u32 pkt_cnt = 0, ret;
1094 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1095 fds.events = POLLOUT;
1097 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {
1098 ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout);
1099 if ((ret || test->fail) && !timeout)
1100 return TEST_FAILURE;
1101 else if (ret == TEST_PASS && timeout)
1105 wait_for_tx_completion(ifobject->xsk);
1109 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1111 int fd = xsk_socket__fd(xsk), err;
1112 socklen_t optlen, expected_len;
1114 optlen = sizeof(*stats);
1115 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1117 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1118 __func__, -err, strerror(-err));
1119 return TEST_FAILURE;
1122 expected_len = sizeof(struct xdp_statistics);
1123 if (optlen != expected_len) {
1124 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1125 __func__, expected_len, optlen);
1126 return TEST_FAILURE;
1132 static int validate_rx_dropped(struct ifobject *ifobject)
1134 struct xsk_socket *xsk = ifobject->xsk->xsk;
1135 struct xdp_statistics stats;
1138 kick_rx(ifobject->xsk);
1140 err = get_xsk_stats(xsk, &stats);
1142 return TEST_FAILURE;
1144 if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2)
1147 return TEST_FAILURE;
1150 static int validate_rx_full(struct ifobject *ifobject)
1152 struct xsk_socket *xsk = ifobject->xsk->xsk;
1153 struct xdp_statistics stats;
1157 kick_rx(ifobject->xsk);
1159 err = get_xsk_stats(xsk, &stats);
1161 return TEST_FAILURE;
1163 if (stats.rx_ring_full)
1166 return TEST_FAILURE;
1169 static int validate_fill_empty(struct ifobject *ifobject)
1171 struct xsk_socket *xsk = ifobject->xsk->xsk;
1172 struct xdp_statistics stats;
1176 kick_rx(ifobject->xsk);
1178 err = get_xsk_stats(xsk, &stats);
1180 return TEST_FAILURE;
1182 if (stats.rx_fill_ring_empty_descs)
1185 return TEST_FAILURE;
1188 static int validate_tx_invalid_descs(struct ifobject *ifobject)
1190 struct xsk_socket *xsk = ifobject->xsk->xsk;
1191 int fd = xsk_socket__fd(xsk);
1192 struct xdp_statistics stats;
1196 optlen = sizeof(stats);
1197 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1199 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1200 __func__, -err, strerror(-err));
1201 return TEST_FAILURE;
1204 if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {
1205 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n",
1206 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
1207 return TEST_FAILURE;
1213 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1214 struct xsk_umem_info *umem, bool tx)
1218 for (i = 0; i < test->nb_sockets; i++) {
1219 bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1222 while (ctr++ < SOCK_RECONF_CTR) {
1223 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1228 /* Retry if it fails as xsk_socket__create() is asynchronous */
1229 if (ctr >= SOCK_RECONF_CTR)
1230 exit_with_error(-ret);
1233 if (ifobject->busy_poll)
1234 enable_busy_poll(&ifobject->xsk_arr[i]);
1238 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1240 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1241 ifobject->xsk = &ifobject->xsk_arr[0];
1242 ifobject->xskmap = test->ifobj_rx->xskmap;
1243 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1246 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
1248 u32 idx = 0, i, buffers_to_fill;
1251 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1252 buffers_to_fill = umem->num_frames;
1254 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1256 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1257 if (ret != buffers_to_fill)
1258 exit_with_error(ENOSPC);
1259 for (i = 0; i < buffers_to_fill; i++) {
1262 if (pkt_stream->use_addr_for_fill) {
1263 struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
1269 addr = i * umem->frame_size;
1272 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1274 xsk_ring_prod__submit(&umem->fq, i);
1277 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1279 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1280 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1281 LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1285 if (ifobject->umem->unaligned_mode)
1286 mmap_flags |= MAP_HUGETLB;
1288 if (ifobject->shared_umem)
1291 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1292 if (bufs == MAP_FAILED)
1293 exit_with_error(errno);
1295 ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz);
1297 exit_with_error(-ret);
1299 xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
1301 xsk_configure_socket(test, ifobject, ifobject->umem, false);
1303 ifobject->xsk = &ifobject->xsk_arr[0];
1305 if (!ifobject->rx_on)
1308 ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);
1310 exit_with_error(errno);
1313 static void *worker_testapp_validate_tx(void *arg)
1315 struct test_spec *test = (struct test_spec *)arg;
1316 struct ifobject *ifobject = test->ifobj_tx;
1319 if (test->current_step == 1) {
1320 if (!ifobject->shared_umem)
1321 thread_common_ops(test, ifobject);
1323 thread_common_ops_tx(test, ifobject);
1326 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
1328 err = send_pkts(test, ifobject);
1330 if (!err && ifobject->validation_func)
1331 err = ifobject->validation_func(ifobject);
1333 report_failure(test);
1338 static void *worker_testapp_validate_rx(void *arg)
1340 struct test_spec *test = (struct test_spec *)arg;
1341 struct ifobject *ifobject = test->ifobj_rx;
1342 struct pollfd fds = { };
1345 if (test->current_step == 1) {
1346 thread_common_ops(test, ifobject);
1348 xsk_clear_xskmap(ifobject->xskmap);
1349 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);
1351 printf("Error: Failed to update xskmap, error %s\n", strerror(-err));
1352 exit_with_error(-err);
1356 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1357 fds.events = POLLIN;
1359 pthread_barrier_wait(&barr);
1361 err = receive_pkts(test, &fds);
1363 if (!err && ifobject->validation_func)
1364 err = ifobject->validation_func(ifobject);
1366 report_failure(test);
1367 pthread_mutex_lock(&pacing_mutex);
1368 pthread_cond_signal(&pacing_cond);
1369 pthread_mutex_unlock(&pacing_mutex);
1375 static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1377 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1379 if (ifobj->shared_umem)
1382 xsk_umem__delete(ifobj->umem->umem);
1383 munmap(ifobj->umem->buffer, umem_sz);
1386 static void handler(int signum)
1391 static bool xdp_prog_changed(struct test_spec *test, struct ifobject *ifobj)
1393 return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode;
1396 static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,
1397 struct bpf_map *xskmap, enum test_mode mode)
1401 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode));
1402 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode));
1404 printf("Error attaching XDP program\n");
1405 exit_with_error(-err);
1408 if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC))
1409 if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) {
1410 ksft_print_msg("ERROR: XDP prog not in DRV mode\n");
1411 exit_with_error(EINVAL);
1414 ifobj->xdp_prog = xdp_prog;
1415 ifobj->xskmap = xskmap;
1419 static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,
1420 struct ifobject *ifobj_tx)
1422 if (xdp_prog_changed(test, ifobj_rx))
1423 xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode);
1425 if (!ifobj_tx || ifobj_tx->shared_umem)
1428 if (xdp_prog_changed(test, ifobj_tx))
1429 xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);
1432 static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,
1433 struct ifobject *ifobj2)
1438 if (pthread_barrier_init(&barr, NULL, 2))
1439 exit_with_error(errno);
1441 test->current_step++;
1442 pkt_stream_reset(ifobj1->pkt_stream);
1445 signal(SIGUSR1, handler);
1446 /*Spawn RX thread */
1447 pthread_create(&t0, NULL, ifobj1->func_ptr, test);
1450 pthread_barrier_wait(&barr);
1451 if (pthread_barrier_destroy(&barr))
1452 exit_with_error(errno);
1454 /*Spawn TX thread */
1455 pthread_create(&t1, NULL, ifobj2->func_ptr, test);
1457 pthread_join(t1, NULL);
1461 pthread_kill(t0, SIGUSR1);
1463 pthread_join(t0, NULL);
1465 if (test->total_steps == test->current_step || test->fail) {
1467 xsk_socket__delete(ifobj2->xsk->xsk);
1468 xsk_socket__delete(ifobj1->xsk->xsk);
1469 testapp_clean_xsk_umem(ifobj1);
1470 if (ifobj2 && !ifobj2->shared_umem)
1471 testapp_clean_xsk_umem(ifobj2);
1474 return !!test->fail;
1477 static int testapp_validate_traffic(struct test_spec *test)
1479 struct ifobject *ifobj_rx = test->ifobj_rx;
1480 struct ifobject *ifobj_tx = test->ifobj_tx;
1482 xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx);
1483 return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);
1486 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)
1488 return __testapp_validate_traffic(test, ifobj, NULL);
1491 static void testapp_teardown(struct test_spec *test)
1495 test_spec_set_name(test, "TEARDOWN");
1496 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1497 if (testapp_validate_traffic(test))
1499 test_spec_reset(test);
1503 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1505 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1506 struct ifobject *tmp_ifobj = (*ifobj1);
1508 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1509 (*ifobj2)->func_ptr = tmp_func_ptr;
1512 *ifobj2 = tmp_ifobj;
1515 static void testapp_bidi(struct test_spec *test)
1517 test_spec_set_name(test, "BIDIRECTIONAL");
1518 test->ifobj_tx->rx_on = true;
1519 test->ifobj_rx->tx_on = true;
1520 test->total_steps = 2;
1521 if (testapp_validate_traffic(test))
1524 print_verbose("Switching Tx/Rx vectors\n");
1525 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1526 __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);
1528 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1531 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
1535 xsk_socket__delete(ifobj_tx->xsk->xsk);
1536 xsk_socket__delete(ifobj_rx->xsk->xsk);
1537 ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
1538 ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
1540 ret = xsk_update_xskmap(ifobj_rx->xskmap, ifobj_rx->xsk->xsk);
1542 exit_with_error(errno);
1545 static void testapp_bpf_res(struct test_spec *test)
1547 test_spec_set_name(test, "BPF_RES");
1548 test->total_steps = 2;
1549 test->nb_sockets = 2;
1550 if (testapp_validate_traffic(test))
1553 swap_xsk_resources(test->ifobj_tx, test->ifobj_rx);
1554 testapp_validate_traffic(test);
1557 static void testapp_headroom(struct test_spec *test)
1559 test_spec_set_name(test, "UMEM_HEADROOM");
1560 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1561 testapp_validate_traffic(test);
1564 static void testapp_stats_rx_dropped(struct test_spec *test)
1566 test_spec_set_name(test, "STAT_RX_DROPPED");
1567 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1568 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1569 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1570 pkt_stream_receive_half(test);
1571 test->ifobj_rx->validation_func = validate_rx_dropped;
1572 testapp_validate_traffic(test);
1575 static void testapp_stats_tx_invalid_descs(struct test_spec *test)
1577 test_spec_set_name(test, "STAT_TX_INVALID");
1578 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
1579 test->ifobj_tx->validation_func = validate_tx_invalid_descs;
1580 testapp_validate_traffic(test);
1583 static void testapp_stats_rx_full(struct test_spec *test)
1585 test_spec_set_name(test, "STAT_RX_FULL");
1586 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1587 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1588 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1589 if (!test->ifobj_rx->pkt_stream)
1590 exit_with_error(ENOMEM);
1592 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
1593 test->ifobj_rx->release_rx = false;
1594 test->ifobj_rx->validation_func = validate_rx_full;
1595 testapp_validate_traffic(test);
1598 static void testapp_stats_fill_empty(struct test_spec *test)
1600 test_spec_set_name(test, "STAT_RX_FILL_EMPTY");
1601 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1602 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1603 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1604 if (!test->ifobj_rx->pkt_stream)
1605 exit_with_error(ENOMEM);
1607 test->ifobj_rx->use_fill_ring = false;
1608 test->ifobj_rx->validation_func = validate_fill_empty;
1609 testapp_validate_traffic(test);
1613 static bool hugepages_present(struct ifobject *ifobject)
1615 const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size;
1618 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1619 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
1620 if (bufs == MAP_FAILED)
1623 munmap(bufs, mmap_sz);
1627 static bool testapp_unaligned(struct test_spec *test)
1629 if (!hugepages_present(test->ifobj_tx)) {
1630 ksft_test_result_skip("No 2M huge pages present.\n");
1634 test_spec_set_name(test, "UNALIGNED_MODE");
1635 test->ifobj_tx->umem->unaligned_mode = true;
1636 test->ifobj_rx->umem->unaligned_mode = true;
1637 /* Let half of the packets straddle a buffer boundrary */
1638 pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2);
1639 test->ifobj_rx->pkt_stream->use_addr_for_fill = true;
1640 testapp_validate_traffic(test);
1645 static void testapp_single_pkt(struct test_spec *test)
1647 struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}};
1649 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1650 testapp_validate_traffic(test);
1653 static void testapp_invalid_desc(struct test_spec *test)
1655 u64 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;
1656 struct pkt pkts[] = {
1657 /* Zero packet address allowed */
1658 {0, PKT_SIZE, 0, true},
1659 /* Allowed packet */
1660 {0x1000, PKT_SIZE, 0, true},
1661 /* Straddling the start of umem */
1662 {-2, PKT_SIZE, 0, false},
1663 /* Packet too large */
1664 {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
1665 /* After umem ends */
1666 {umem_size, PKT_SIZE, 0, false},
1667 /* Straddle the end of umem */
1668 {umem_size - PKT_SIZE / 2, PKT_SIZE, 0, false},
1669 /* Straddle a page boundrary */
1670 {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false},
1671 /* Straddle a 2K boundrary */
1672 {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true},
1673 /* Valid packet for synch so that something is received */
1674 {0x4000, PKT_SIZE, 0, true}};
1676 if (test->ifobj_tx->umem->unaligned_mode) {
1677 /* Crossing a page boundrary allowed */
1678 pkts[6].valid = true;
1680 if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
1681 /* Crossing a 2K frame size boundrary not allowed */
1682 pkts[7].valid = false;
1685 if (test->ifobj_tx->shared_umem) {
1686 pkts[4].addr += umem_size;
1687 pkts[5].addr += umem_size;
1690 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1691 testapp_validate_traffic(test);
1694 static void testapp_xdp_drop(struct test_spec *test)
1696 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
1697 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
1699 test_spec_set_name(test, "XDP_DROP_HALF");
1700 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop,
1701 skel_rx->maps.xsk, skel_tx->maps.xsk);
1703 pkt_stream_receive_half(test);
1704 testapp_validate_traffic(test);
1707 static void testapp_xdp_metadata_count(struct test_spec *test)
1709 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
1710 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
1711 struct bpf_map *data_map;
1715 test_spec_set_name(test, "XDP_METADATA_COUNT");
1716 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata,
1717 skel_tx->progs.xsk_xdp_populate_metadata,
1718 skel_rx->maps.xsk, skel_tx->maps.xsk);
1719 test->ifobj_rx->use_metadata = true;
1721 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");
1722 if (!data_map || !bpf_map__is_internal(data_map))
1723 exit_with_error(ENOMEM);
1725 if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY))
1726 exit_with_error(errno);
1728 testapp_validate_traffic(test);
1731 static void testapp_poll_txq_tmout(struct test_spec *test)
1733 test_spec_set_name(test, "POLL_TXQ_FULL");
1735 test->ifobj_tx->use_poll = true;
1736 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */
1737 test->ifobj_tx->umem->frame_size = 2048;
1738 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
1739 testapp_validate_traffic_single_thread(test, test->ifobj_tx);
1742 static void testapp_poll_rxq_tmout(struct test_spec *test)
1744 test_spec_set_name(test, "POLL_RXQ_EMPTY");
1745 test->ifobj_rx->use_poll = true;
1746 testapp_validate_traffic_single_thread(test, test->ifobj_rx);
1749 static int xsk_load_xdp_programs(struct ifobject *ifobj)
1751 ifobj->xdp_progs = xsk_xdp_progs__open_and_load();
1752 if (libbpf_get_error(ifobj->xdp_progs))
1753 return libbpf_get_error(ifobj->xdp_progs);
1758 static void xsk_unload_xdp_programs(struct ifobject *ifobj)
1760 xsk_xdp_progs__destroy(ifobj->xdp_progs);
1763 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
1764 const char *dst_ip, const char *src_ip, const u16 dst_port,
1765 const u16 src_port, thread_func_t func_ptr)
1770 memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
1771 memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
1773 inet_aton(dst_ip, &ip);
1774 ifobj->dst_ip = ip.s_addr;
1776 inet_aton(src_ip, &ip);
1777 ifobj->src_ip = ip.s_addr;
1779 ifobj->dst_port = dst_port;
1780 ifobj->src_port = src_port;
1782 ifobj->func_ptr = func_ptr;
1784 err = xsk_load_xdp_programs(ifobj);
1786 printf("Error loading XDP program\n");
1787 exit_with_error(err);
1791 static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type)
1794 case TEST_TYPE_STATS_RX_DROPPED:
1795 if (mode == TEST_MODE_ZC) {
1796 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
1799 testapp_stats_rx_dropped(test);
1801 case TEST_TYPE_STATS_TX_INVALID_DESCS:
1802 testapp_stats_tx_invalid_descs(test);
1804 case TEST_TYPE_STATS_RX_FULL:
1805 testapp_stats_rx_full(test);
1807 case TEST_TYPE_STATS_FILL_EMPTY:
1808 testapp_stats_fill_empty(test);
1810 case TEST_TYPE_TEARDOWN:
1811 testapp_teardown(test);
1813 case TEST_TYPE_BIDI:
1816 case TEST_TYPE_BPF_RES:
1817 testapp_bpf_res(test);
1819 case TEST_TYPE_RUN_TO_COMPLETION:
1820 test_spec_set_name(test, "RUN_TO_COMPLETION");
1821 testapp_validate_traffic(test);
1823 case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT:
1824 test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT");
1825 testapp_single_pkt(test);
1827 case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME:
1828 test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE");
1829 test->ifobj_tx->umem->frame_size = 2048;
1830 test->ifobj_rx->umem->frame_size = 2048;
1831 pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE);
1832 testapp_validate_traffic(test);
1834 case TEST_TYPE_RX_POLL:
1835 test->ifobj_rx->use_poll = true;
1836 test_spec_set_name(test, "POLL_RX");
1837 testapp_validate_traffic(test);
1839 case TEST_TYPE_TX_POLL:
1840 test->ifobj_tx->use_poll = true;
1841 test_spec_set_name(test, "POLL_TX");
1842 testapp_validate_traffic(test);
1844 case TEST_TYPE_POLL_TXQ_TMOUT:
1845 testapp_poll_txq_tmout(test);
1847 case TEST_TYPE_POLL_RXQ_TMOUT:
1848 testapp_poll_rxq_tmout(test);
1850 case TEST_TYPE_ALIGNED_INV_DESC:
1851 test_spec_set_name(test, "ALIGNED_INV_DESC");
1852 testapp_invalid_desc(test);
1854 case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME:
1855 test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE");
1856 test->ifobj_tx->umem->frame_size = 2048;
1857 test->ifobj_rx->umem->frame_size = 2048;
1858 testapp_invalid_desc(test);
1860 case TEST_TYPE_UNALIGNED_INV_DESC:
1861 if (!hugepages_present(test->ifobj_tx)) {
1862 ksft_test_result_skip("No 2M huge pages present.\n");
1865 test_spec_set_name(test, "UNALIGNED_INV_DESC");
1866 test->ifobj_tx->umem->unaligned_mode = true;
1867 test->ifobj_rx->umem->unaligned_mode = true;
1868 testapp_invalid_desc(test);
1870 case TEST_TYPE_UNALIGNED:
1871 if (!testapp_unaligned(test))
1874 case TEST_TYPE_HEADROOM:
1875 testapp_headroom(test);
1877 case TEST_TYPE_XDP_DROP_HALF:
1878 testapp_xdp_drop(test);
1880 case TEST_TYPE_XDP_METADATA_COUNT:
1881 testapp_xdp_metadata_count(test);
1888 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
1890 pkt_stream_restore_default(test);
1893 static struct ifobject *ifobject_create(void)
1895 struct ifobject *ifobj;
1897 ifobj = calloc(1, sizeof(struct ifobject));
1901 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
1902 if (!ifobj->xsk_arr)
1905 ifobj->umem = calloc(1, sizeof(*ifobj->umem));
1912 free(ifobj->xsk_arr);
1918 static void ifobject_delete(struct ifobject *ifobj)
1921 free(ifobj->xsk_arr);
1925 static bool is_xdp_supported(int ifindex)
1927 int flags = XDP_FLAGS_DRV_MODE;
1929 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
1930 struct bpf_insn insns[2] = {
1931 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
1934 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
1937 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
1941 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
1947 bpf_xdp_detach(ifindex, flags, NULL);
1953 int main(int argc, char **argv)
1955 struct pkt_stream *rx_pkt_stream_default;
1956 struct pkt_stream *tx_pkt_stream_default;
1957 struct ifobject *ifobj_tx, *ifobj_rx;
1958 int modes = TEST_MODE_SKB + 1;
1959 u32 i, j, failed_tests = 0;
1960 struct test_spec test;
1963 /* Use libbpf 1.0 API mode */
1964 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1966 ifobj_tx = ifobject_create();
1968 exit_with_error(ENOMEM);
1969 ifobj_rx = ifobject_create();
1971 exit_with_error(ENOMEM);
1973 setlocale(LC_ALL, "");
1975 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
1977 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
1978 ifobj_tx->shared_umem = shared_netdev;
1979 ifobj_rx->shared_umem = shared_netdev;
1981 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
1982 usage(basename(argv[0]));
1986 if (is_xdp_supported(ifobj_tx->ifindex)) {
1988 if (ifobj_zc_avail(ifobj_tx))
1992 init_iface(ifobj_rx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2,
1993 worker_testapp_validate_rx);
1994 init_iface(ifobj_tx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
1995 worker_testapp_validate_tx);
1997 test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
1998 tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1999 rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
2000 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
2001 exit_with_error(ENOMEM);
2002 test.tx_pkt_stream_default = tx_pkt_stream_default;
2003 test.rx_pkt_stream_default = rx_pkt_stream_default;
2005 ksft_set_plan(modes * TEST_TYPE_MAX);
2007 for (i = 0; i < modes; i++) {
2008 for (j = 0; j < TEST_TYPE_MAX; j++) {
2009 test_spec_init(&test, ifobj_tx, ifobj_rx, i);
2010 run_pkt_test(&test, i, j);
2018 pkt_stream_delete(tx_pkt_stream_default);
2019 pkt_stream_delete(rx_pkt_stream_default);
2020 xsk_unload_xdp_programs(ifobj_tx);
2021 xsk_unload_xdp_programs(ifobj_rx);
2022 ifobject_delete(ifobj_tx);
2023 ifobject_delete(ifobj_rx);