1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2020 Intel Corporation. */
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
57 * - Single process spawns two threads: Tx and Rx
58 * - Each of these two threads attach to a veth interface within their assigned
60 * - Each thread Creates one AF_XDP socket connected to a unique umem for each
62 * - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy>
63 * - Rx thread verifies if all 10k packets were received and delivered in-order,
64 * and have the right content
66 * Enable/disable packet dump mode:
67 * --------------------------
68 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
69 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
76 #include <asm/barrier.h>
77 #include <linux/if_link.h>
78 #include <linux/if_ether.h>
80 #include <linux/udp.h>
81 #include <arpa/inet.h>
93 #include <sys/socket.h>
95 #include <sys/types.h>
96 #include <sys/queue.h>
99 #include <stdatomic.h>
101 #include "xskxceiver.h"
103 #include <linux/filter.h>
104 #include "../kselftest.h"
106 /* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf.
107 * Until xskxceiver is either moved or re-writed into libxdp, suppress
108 * deprecation warnings in this file
110 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
112 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
113 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
114 static const char *IP1 = "192.168.100.162";
115 static const char *IP2 = "192.168.100.161";
116 static const u16 UDP_PORT1 = 2020;
117 static const u16 UDP_PORT2 = 2121;
119 static void __exit_with_error(int error, const char *file, const char *func, int line)
121 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
126 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
128 #define mode_string(test) (test)->ifobj_tx->xdp_flags & XDP_FLAGS_SKB_MODE ? "SKB" : "DRV"
129 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
131 static void report_failure(struct test_spec *test)
136 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
141 static void memset32_htonl(void *dest, u32 val, u32 size)
143 u32 *ptr = (u32 *)dest;
148 for (i = 0; i < (size & (~0x3)); i += 4)
153 * Fold a partial checksum
154 * This function code has been taken from
155 * Linux kernel include/asm-generic/checksum.h
157 static __u16 csum_fold(__u32 csum)
159 u32 sum = (__force u32)csum;
161 sum = (sum & 0xffff) + (sum >> 16);
162 sum = (sum & 0xffff) + (sum >> 16);
163 return (__force __u16)~sum;
167 * This function code has been taken from
168 * Linux kernel lib/checksum.c
170 static u32 from64to32(u64 x)
172 /* add up 32-bit and 32-bit for 32+c bit */
173 x = (x & 0xffffffff) + (x >> 32);
175 x = (x & 0xffffffff) + (x >> 32);
180 * This function code has been taken from
181 * Linux kernel lib/checksum.c
183 static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
185 unsigned long long s = (__force u32)sum;
187 s += (__force u32)saddr;
188 s += (__force u32)daddr;
189 #ifdef __BIG_ENDIAN__
192 s += (proto + len) << 8;
194 return (__force __u32)from64to32(s);
198 * This function has been taken from
199 * Linux kernel include/asm-generic/checksum.h
201 static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
203 return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
206 static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
211 /* udp hdr and data */
212 for (; cnt < len; cnt += 2)
213 csum += udp_pkt[cnt >> 1];
215 return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
218 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
220 memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
221 memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
222 eth_hdr->h_proto = htons(ETH_P_IP);
225 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)
227 ip_hdr->version = IP_PKT_VER;
229 ip_hdr->tos = IP_PKT_TOS;
230 ip_hdr->tot_len = htons(IP_PKT_SIZE);
232 ip_hdr->frag_off = 0;
233 ip_hdr->ttl = IPDEFTTL;
234 ip_hdr->protocol = IPPROTO_UDP;
235 ip_hdr->saddr = ifobject->src_ip;
236 ip_hdr->daddr = ifobject->dst_ip;
240 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject,
241 struct udphdr *udp_hdr)
243 udp_hdr->source = htons(ifobject->src_port);
244 udp_hdr->dest = htons(ifobject->dst_port);
245 udp_hdr->len = htons(UDP_PKT_SIZE);
246 memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE);
249 static bool is_umem_valid(struct ifobject *ifobj)
251 return !!ifobj->umem->umem;
254 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
258 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);
261 static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size)
263 struct xsk_umem_config cfg = {
264 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
265 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
266 .frame_size = umem->frame_size,
267 .frame_headroom = umem->frame_headroom,
268 .flags = XSK_UMEM__DEFAULT_FLAGS
272 if (umem->unaligned_mode)
273 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
275 ret = xsk_umem__create(&umem->umem, buffer, size,
276 &umem->fq, &umem->cq, &cfg);
280 umem->buffer = buffer;
284 static void enable_busy_poll(struct xsk_socket_info *xsk)
289 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
290 (void *)&sock_opt, sizeof(sock_opt)) < 0)
291 exit_with_error(errno);
294 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
295 (void *)&sock_opt, sizeof(sock_opt)) < 0)
296 exit_with_error(errno);
298 sock_opt = BATCH_SIZE;
299 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
300 (void *)&sock_opt, sizeof(sock_opt)) < 0)
301 exit_with_error(errno);
304 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
305 struct ifobject *ifobject, bool shared)
307 struct xsk_socket_config cfg = {};
308 struct xsk_ring_cons *rxr;
309 struct xsk_ring_prod *txr;
312 cfg.rx_size = xsk->rxqsize;
313 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
314 cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
315 cfg.xdp_flags = ifobject->xdp_flags;
316 cfg.bind_flags = ifobject->bind_flags;
318 cfg.bind_flags |= XDP_SHARED_UMEM;
320 txr = ifobject->tx_on ? &xsk->tx : NULL;
321 rxr = ifobject->rx_on ? &xsk->rx : NULL;
322 return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);
325 static struct option long_options[] = {
326 {"interface", required_argument, 0, 'i'},
327 {"busy-poll", no_argument, 0, 'b'},
328 {"dump-pkts", no_argument, 0, 'D'},
329 {"verbose", no_argument, 0, 'v'},
333 static void usage(const char *prog)
336 " Usage: %s [OPTIONS]\n"
338 " -i, --interface Use interface\n"
339 " -D, --dump-pkts Dump packets L2 - L5\n"
340 " -v, --verbose Verbose output\n"
341 " -b, --busy-poll Enable busy poll\n";
343 ksft_print_msg(str, prog);
346 static int switch_namespace(const char *nsname)
348 char fqns[26] = "/var/run/netns/";
351 if (!nsname || strlen(nsname) == 0)
354 strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1);
355 nsfd = open(fqns, O_RDONLY);
358 exit_with_error(errno);
360 if (setns(nsfd, 0) == -1)
361 exit_with_error(errno);
363 print_verbose("NS switched: %s\n", nsname);
368 static bool validate_interface(struct ifobject *ifobj)
370 if (!strcmp(ifobj->ifname, ""))
375 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
378 struct ifobject *ifobj;
379 u32 interface_nb = 0;
387 c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index);
393 if (interface_nb == 0)
395 else if (interface_nb == 1)
400 sptr = strndupa(optarg, strlen(optarg));
401 memcpy(ifobj->ifname, strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS);
402 token = strsep(&sptr, ",");
404 memcpy(ifobj->nsname, token, MAX_INTERFACES_NAMESPACE_CHARS);
414 ifobj_tx->busy_poll = true;
415 ifobj_rx->busy_poll = true;
418 usage(basename(argv[0]));
424 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
425 struct ifobject *ifobj_rx)
429 for (i = 0; i < MAX_INTERFACES; i++) {
430 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
432 ifobj->xsk = &ifobj->xsk_arr[0];
433 ifobj->use_poll = false;
434 ifobj->use_fill_ring = true;
435 ifobj->release_rx = true;
436 ifobj->validation_func = NULL;
439 ifobj->rx_on = false;
441 ifobj->pkt_stream = test->tx_pkt_stream_default;
444 ifobj->tx_on = false;
445 ifobj->pkt_stream = test->rx_pkt_stream_default;
448 memset(ifobj->umem, 0, sizeof(*ifobj->umem));
449 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
450 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
451 if (ifobj->shared_umem && ifobj->rx_on)
452 ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS *
453 XSK_UMEM__DEFAULT_FRAME_SIZE;
455 for (j = 0; j < MAX_SOCKETS; j++) {
456 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
457 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
461 test->ifobj_tx = ifobj_tx;
462 test->ifobj_rx = ifobj_rx;
463 test->current_step = 0;
464 test->total_steps = 1;
465 test->nb_sockets = 1;
469 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
470 struct ifobject *ifobj_rx, enum test_mode mode)
472 struct pkt_stream *tx_pkt_stream;
473 struct pkt_stream *rx_pkt_stream;
476 tx_pkt_stream = test->tx_pkt_stream_default;
477 rx_pkt_stream = test->rx_pkt_stream_default;
478 memset(test, 0, sizeof(*test));
479 test->tx_pkt_stream_default = tx_pkt_stream;
480 test->rx_pkt_stream_default = rx_pkt_stream;
482 for (i = 0; i < MAX_INTERFACES; i++) {
483 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
485 ifobj->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
486 if (mode == TEST_MODE_SKB)
487 ifobj->xdp_flags |= XDP_FLAGS_SKB_MODE;
489 ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE;
491 ifobj->bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY;
494 __test_spec_init(test, ifobj_tx, ifobj_rx);
497 static void test_spec_reset(struct test_spec *test)
499 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
502 static void test_spec_set_name(struct test_spec *test, const char *name)
504 strncpy(test->name, name, MAX_TEST_NAME_SIZE);
507 static void pkt_stream_reset(struct pkt_stream *pkt_stream)
510 pkt_stream->rx_pkt_nb = 0;
513 static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb)
515 if (pkt_nb >= pkt_stream->nb_pkts)
518 return &pkt_stream->pkts[pkt_nb];
521 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
523 while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) {
525 if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid)
526 return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++];
527 pkt_stream->rx_pkt_nb++;
532 static void pkt_stream_delete(struct pkt_stream *pkt_stream)
534 free(pkt_stream->pkts);
538 static void pkt_stream_restore_default(struct test_spec *test)
540 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
541 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
543 if (tx_pkt_stream != test->tx_pkt_stream_default) {
544 pkt_stream_delete(test->ifobj_tx->pkt_stream);
545 test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
548 if (rx_pkt_stream != test->rx_pkt_stream_default) {
549 pkt_stream_delete(test->ifobj_rx->pkt_stream);
550 test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
554 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
556 struct pkt_stream *pkt_stream;
558 pkt_stream = calloc(1, sizeof(*pkt_stream));
562 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
563 if (!pkt_stream->pkts) {
568 pkt_stream->nb_pkts = nb_pkts;
572 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len)
574 pkt->addr = addr + umem->base_addr;
576 if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)
582 static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
584 struct pkt_stream *pkt_stream;
587 pkt_stream = __pkt_stream_alloc(nb_pkts);
589 exit_with_error(ENOMEM);
591 pkt_stream->nb_pkts = nb_pkts;
592 for (i = 0; i < nb_pkts; i++) {
593 pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size,
595 pkt_stream->pkts[i].payload = i;
601 static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
602 struct pkt_stream *pkt_stream)
604 return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
607 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
609 struct pkt_stream *pkt_stream;
611 pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
612 test->ifobj_tx->pkt_stream = pkt_stream;
613 pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
614 test->ifobj_rx->pkt_stream = pkt_stream;
617 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
620 struct xsk_umem_info *umem = ifobj->umem;
621 struct pkt_stream *pkt_stream;
624 pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
625 for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
626 pkt_set(umem, &pkt_stream->pkts[i],
627 (i % umem->num_frames) * umem->frame_size + offset, pkt_len);
629 ifobj->pkt_stream = pkt_stream;
632 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
634 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
635 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
638 static void pkt_stream_receive_half(struct test_spec *test)
640 struct xsk_umem_info *umem = test->ifobj_rx->umem;
641 struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream;
644 test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts,
645 pkt_stream->pkts[0].len);
646 pkt_stream = test->ifobj_rx->pkt_stream;
647 for (i = 1; i < pkt_stream->nb_pkts; i += 2)
648 pkt_stream->pkts[i].valid = false;
651 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)
653 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);
654 struct udphdr *udp_hdr;
655 struct ethhdr *eth_hdr;
656 struct iphdr *ip_hdr;
661 if (!pkt->valid || pkt->len < MIN_PKT_SIZE)
664 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);
665 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr));
666 ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr));
667 eth_hdr = (struct ethhdr *)data;
669 gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr);
670 gen_ip_hdr(ifobject, ip_hdr);
671 gen_udp_csum(udp_hdr, ip_hdr);
672 gen_eth_hdr(ifobject, eth_hdr);
677 static void __pkt_stream_generate_custom(struct ifobject *ifobj,
678 struct pkt *pkts, u32 nb_pkts)
680 struct pkt_stream *pkt_stream;
683 pkt_stream = __pkt_stream_alloc(nb_pkts);
685 exit_with_error(ENOMEM);
687 for (i = 0; i < nb_pkts; i++) {
688 pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr;
689 pkt_stream->pkts[i].len = pkts[i].len;
690 pkt_stream->pkts[i].payload = i;
691 pkt_stream->pkts[i].valid = pkts[i].valid;
694 ifobj->pkt_stream = pkt_stream;
697 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
699 __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts);
700 __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts);
703 static void pkt_dump(void *pkt, u32 len)
705 char s[INET_ADDRSTRLEN];
706 struct ethhdr *ethhdr;
707 struct udphdr *udphdr;
712 iphdr = pkt + sizeof(*ethhdr);
713 udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr);
715 /*extract L2 frame */
716 fprintf(stdout, "DEBUG>> L2: dst mac: ");
717 for (i = 0; i < ETH_ALEN; i++)
718 fprintf(stdout, "%02X", ethhdr->h_dest[i]);
720 fprintf(stdout, "\nDEBUG>> L2: src mac: ");
721 for (i = 0; i < ETH_ALEN; i++)
722 fprintf(stdout, "%02X", ethhdr->h_source[i]);
724 /*extract L3 frame */
725 fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);
726 fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",
727 inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));
728 fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",
729 inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));
730 /*extract L4 frame */
731 fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));
732 fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));
733 /*extract L5 frame */
734 payload = *((uint32_t *)(pkt + PKT_HDR_SIZE));
736 fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload);
737 fprintf(stdout, "---------------------------------------\n");
740 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr,
743 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
744 u32 offset = addr % umem->frame_size, expected_offset = 0;
746 if (!pkt_stream->use_addr_for_fill)
749 expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
751 if (offset == expected_offset)
754 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
758 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
760 void *data = xsk_umem__get_data(buffer, addr);
761 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr));
764 ksft_print_msg("[%s] too many packets received\n", __func__);
768 if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) {
769 /* Do not try to verify packets that are smaller than minimum size. */
773 if (pkt->len != len) {
774 ksft_print_msg("[%s] expected length [%d], got length [%d]\n",
775 __func__, pkt->len, len);
779 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
780 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE)));
783 pkt_dump(data, PKT_SIZE);
785 if (pkt->payload != seqnum) {
786 ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n",
787 __func__, pkt->payload, seqnum);
791 ksft_print_msg("Invalid frame received: ");
792 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,
800 static void kick_tx(struct xsk_socket_info *xsk)
804 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
807 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
811 exit_with_error(errno);
814 static void kick_rx(struct xsk_socket_info *xsk)
818 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
820 exit_with_error(errno);
823 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
828 if (xsk_ring_prod__needs_wakeup(&xsk->tx))
831 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
833 if (rcvd > xsk->outstanding_tx) {
834 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
836 ksft_print_msg("[%s] Too many packets completed\n", __func__);
837 ksft_print_msg("Last completion address: %llx\n", addr);
841 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
842 xsk->outstanding_tx -= rcvd;
848 static int receive_pkts(struct test_spec *test, struct pollfd *fds)
850 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
851 struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream;
852 u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0;
853 struct xsk_socket_info *xsk = test->ifobj_rx->xsk;
854 struct ifobject *ifobj = test->ifobj_rx;
855 struct xsk_umem_info *umem = xsk->umem;
859 ret = gettimeofday(&tv_now, NULL);
861 exit_with_error(errno);
862 timeradd(&tv_now, &tv_timeout, &tv_end);
864 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
866 ret = gettimeofday(&tv_now, NULL);
868 exit_with_error(errno);
869 if (timercmp(&tv_now, &tv_end, >)) {
870 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
875 if (ifobj->use_poll) {
876 ret = poll(fds, 1, POLL_TMOUT);
878 exit_with_error(-ret);
881 if (!is_umem_valid(test->ifobj_tx))
884 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
889 if (!(fds->revents & POLLIN))
893 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
897 if (ifobj->use_fill_ring) {
898 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
899 while (ret != rcvd) {
901 exit_with_error(-ret);
902 if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
903 ret = poll(fds, 1, POLL_TMOUT);
905 exit_with_error(-ret);
907 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
911 for (i = 0; i < rcvd; i++) {
912 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
913 u64 addr = desc->addr, orig;
915 orig = xsk_umem__extract_addr(addr);
916 addr = xsk_umem__add_offset_to_addr(addr);
918 if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) ||
919 !is_offset_correct(umem, pkt_stream, addr, pkt->addr))
922 if (ifobj->use_fill_ring)
923 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
924 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
927 if (ifobj->use_fill_ring)
928 xsk_ring_prod__submit(&umem->fq, rcvd);
929 if (ifobj->release_rx)
930 xsk_ring_cons__release(&xsk->rx, rcvd);
932 pthread_mutex_lock(&pacing_mutex);
933 pkts_in_flight -= pkts_sent;
934 if (pkts_in_flight < umem->num_frames)
935 pthread_cond_signal(&pacing_cond);
936 pthread_mutex_unlock(&pacing_mutex);
943 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds,
946 struct xsk_socket_info *xsk = ifobject->xsk;
947 bool use_poll = ifobject->use_poll;
948 u32 i, idx = 0, ret, valid_pkts = 0;
950 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
952 ret = poll(fds, 1, POLL_TMOUT);
955 ksft_print_msg("ERROR: [%s] Poll error %d\n",
964 ksft_print_msg("ERROR: [%s] Poll error %d\n",
970 complete_pkts(xsk, BATCH_SIZE);
973 for (i = 0; i < BATCH_SIZE; i++) {
974 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
975 struct pkt *pkt = pkt_generate(ifobject, *pkt_nb);
980 tx_desc->addr = pkt->addr;
981 tx_desc->len = pkt->len;
987 pthread_mutex_lock(&pacing_mutex);
988 pkts_in_flight += valid_pkts;
989 /* pkts_in_flight might be negative if many invalid packets are sent */
990 if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) {
992 pthread_cond_wait(&pacing_cond, &pacing_mutex);
994 pthread_mutex_unlock(&pacing_mutex);
996 xsk_ring_prod__submit(&xsk->tx, i);
997 xsk->outstanding_tx += valid_pkts;
1000 ret = poll(fds, 1, POLL_TMOUT);
1002 if (ret == 0 && timeout)
1005 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1006 return TEST_FAILURE;
1011 if (complete_pkts(xsk, i))
1012 return TEST_FAILURE;
1018 return TEST_CONTINUE;
1021 static void wait_for_tx_completion(struct xsk_socket_info *xsk)
1023 while (xsk->outstanding_tx)
1024 complete_pkts(xsk, BATCH_SIZE);
1027 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1029 bool timeout = !is_umem_valid(test->ifobj_rx);
1030 struct pollfd fds = { };
1031 u32 pkt_cnt = 0, ret;
1033 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1034 fds.events = POLLOUT;
1036 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {
1037 ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout);
1038 if ((ret || test->fail) && !timeout)
1039 return TEST_FAILURE;
1040 else if (ret == TEST_PASS && timeout)
1044 wait_for_tx_completion(ifobject->xsk);
1048 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1050 int fd = xsk_socket__fd(xsk), err;
1051 socklen_t optlen, expected_len;
1053 optlen = sizeof(*stats);
1054 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1056 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1057 __func__, -err, strerror(-err));
1058 return TEST_FAILURE;
1061 expected_len = sizeof(struct xdp_statistics);
1062 if (optlen != expected_len) {
1063 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1064 __func__, expected_len, optlen);
1065 return TEST_FAILURE;
1071 static int validate_rx_dropped(struct ifobject *ifobject)
1073 struct xsk_socket *xsk = ifobject->xsk->xsk;
1074 struct xdp_statistics stats;
1077 kick_rx(ifobject->xsk);
1079 err = get_xsk_stats(xsk, &stats);
1081 return TEST_FAILURE;
1083 if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2)
1086 return TEST_FAILURE;
1089 static int validate_rx_full(struct ifobject *ifobject)
1091 struct xsk_socket *xsk = ifobject->xsk->xsk;
1092 struct xdp_statistics stats;
1096 kick_rx(ifobject->xsk);
1098 err = get_xsk_stats(xsk, &stats);
1100 return TEST_FAILURE;
1102 if (stats.rx_ring_full)
1105 return TEST_FAILURE;
1108 static int validate_fill_empty(struct ifobject *ifobject)
1110 struct xsk_socket *xsk = ifobject->xsk->xsk;
1111 struct xdp_statistics stats;
1115 kick_rx(ifobject->xsk);
1117 err = get_xsk_stats(xsk, &stats);
1119 return TEST_FAILURE;
1121 if (stats.rx_fill_ring_empty_descs)
1124 return TEST_FAILURE;
1127 static int validate_tx_invalid_descs(struct ifobject *ifobject)
1129 struct xsk_socket *xsk = ifobject->xsk->xsk;
1130 int fd = xsk_socket__fd(xsk);
1131 struct xdp_statistics stats;
1135 optlen = sizeof(stats);
1136 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1138 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1139 __func__, -err, strerror(-err));
1140 return TEST_FAILURE;
1143 if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {
1144 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n",
1145 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
1146 return TEST_FAILURE;
1152 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1153 struct xsk_umem_info *umem, bool tx)
1157 for (i = 0; i < test->nb_sockets; i++) {
1158 bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1161 while (ctr++ < SOCK_RECONF_CTR) {
1162 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1167 /* Retry if it fails as xsk_socket__create() is asynchronous */
1168 if (ctr >= SOCK_RECONF_CTR)
1169 exit_with_error(-ret);
1172 if (ifobject->busy_poll)
1173 enable_busy_poll(&ifobject->xsk_arr[i]);
1177 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1179 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1180 ifobject->xsk = &ifobject->xsk_arr[0];
1181 ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd;
1182 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1185 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
1187 u32 idx = 0, i, buffers_to_fill;
1190 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1191 buffers_to_fill = umem->num_frames;
1193 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1195 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1196 if (ret != buffers_to_fill)
1197 exit_with_error(ENOSPC);
1198 for (i = 0; i < buffers_to_fill; i++) {
1201 if (pkt_stream->use_addr_for_fill) {
1202 struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
1208 addr = i * umem->frame_size;
1211 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1213 xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
1216 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1218 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1219 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1220 LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1224 ifobject->ns_fd = switch_namespace(ifobject->nsname);
1226 if (ifobject->umem->unaligned_mode)
1227 mmap_flags |= MAP_HUGETLB;
1229 if (ifobject->shared_umem)
1232 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1233 if (bufs == MAP_FAILED)
1234 exit_with_error(errno);
1236 ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz);
1238 exit_with_error(-ret);
1240 xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
1242 xsk_configure_socket(test, ifobject, ifobject->umem, false);
1244 ifobject->xsk = &ifobject->xsk_arr[0];
1246 if (!ifobject->rx_on)
1249 ifindex = if_nametoindex(ifobject->ifname);
1251 exit_with_error(errno);
1253 ret = xsk_setup_xdp_prog_xsk(ifobject->xsk->xsk, &ifobject->xsk_map_fd);
1255 exit_with_error(-ret);
1257 ret = bpf_xdp_query(ifindex, ifobject->xdp_flags, &opts);
1259 exit_with_error(-ret);
1261 if (ifobject->xdp_flags & XDP_FLAGS_SKB_MODE) {
1262 if (opts.attach_mode != XDP_ATTACHED_SKB) {
1263 ksft_print_msg("ERROR: [%s] XDP prog not in SKB mode\n");
1264 exit_with_error(-EINVAL);
1266 } else if (ifobject->xdp_flags & XDP_FLAGS_DRV_MODE) {
1267 if (opts.attach_mode != XDP_ATTACHED_DRV) {
1268 ksft_print_msg("ERROR: [%s] XDP prog not in DRV mode\n");
1269 exit_with_error(-EINVAL);
1273 ret = xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
1275 exit_with_error(-ret);
1278 static void *worker_testapp_validate_tx(void *arg)
1280 struct test_spec *test = (struct test_spec *)arg;
1281 struct ifobject *ifobject = test->ifobj_tx;
1284 if (test->current_step == 1) {
1285 if (!ifobject->shared_umem)
1286 thread_common_ops(test, ifobject);
1288 thread_common_ops_tx(test, ifobject);
1291 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
1293 err = send_pkts(test, ifobject);
1295 if (!err && ifobject->validation_func)
1296 err = ifobject->validation_func(ifobject);
1298 report_failure(test);
1303 static void *worker_testapp_validate_rx(void *arg)
1305 struct test_spec *test = (struct test_spec *)arg;
1306 struct ifobject *ifobject = test->ifobj_rx;
1307 struct pollfd fds = { };
1311 if (test->current_step == 1) {
1312 thread_common_ops(test, ifobject);
1314 bpf_map_delete_elem(ifobject->xsk_map_fd, &id);
1315 xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
1318 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1319 fds.events = POLLIN;
1321 pthread_barrier_wait(&barr);
1323 err = receive_pkts(test, &fds);
1325 if (!err && ifobject->validation_func)
1326 err = ifobject->validation_func(ifobject);
1328 report_failure(test);
1329 pthread_mutex_lock(&pacing_mutex);
1330 pthread_cond_signal(&pacing_cond);
1331 pthread_mutex_unlock(&pacing_mutex);
1337 static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1339 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1341 if (ifobj->shared_umem)
1344 xsk_umem__delete(ifobj->umem->umem);
1345 munmap(ifobj->umem->buffer, umem_sz);
1348 static void handler(int signum)
1353 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj,
1354 enum test_type type)
1356 bool old_shared_umem = ifobj->shared_umem;
1359 if (pthread_barrier_init(&barr, NULL, 2))
1360 exit_with_error(errno);
1362 test->current_step++;
1363 if (type == TEST_TYPE_POLL_RXQ_TMOUT)
1364 pkt_stream_reset(ifobj->pkt_stream);
1367 test->ifobj_rx->shared_umem = false;
1368 test->ifobj_tx->shared_umem = false;
1370 signal(SIGUSR1, handler);
1372 pthread_create(&t0, NULL, ifobj->func_ptr, test);
1374 if (type != TEST_TYPE_POLL_TXQ_TMOUT)
1375 pthread_barrier_wait(&barr);
1377 if (pthread_barrier_destroy(&barr))
1378 exit_with_error(errno);
1380 pthread_kill(t0, SIGUSR1);
1381 pthread_join(t0, NULL);
1383 if (test->total_steps == test->current_step || test->fail) {
1384 xsk_socket__delete(ifobj->xsk->xsk);
1385 testapp_clean_xsk_umem(ifobj);
1388 test->ifobj_rx->shared_umem = old_shared_umem;
1389 test->ifobj_tx->shared_umem = old_shared_umem;
1391 return !!test->fail;
1394 static int testapp_validate_traffic(struct test_spec *test)
1396 struct ifobject *ifobj_tx = test->ifobj_tx;
1397 struct ifobject *ifobj_rx = test->ifobj_rx;
1400 if (pthread_barrier_init(&barr, NULL, 2))
1401 exit_with_error(errno);
1403 test->current_step++;
1404 pkt_stream_reset(ifobj_rx->pkt_stream);
1407 /*Spawn RX thread */
1408 pthread_create(&t0, NULL, ifobj_rx->func_ptr, test);
1410 pthread_barrier_wait(&barr);
1411 if (pthread_barrier_destroy(&barr))
1412 exit_with_error(errno);
1414 /*Spawn TX thread */
1415 pthread_create(&t1, NULL, ifobj_tx->func_ptr, test);
1417 pthread_join(t1, NULL);
1418 pthread_join(t0, NULL);
1420 if (test->total_steps == test->current_step || test->fail) {
1421 xsk_socket__delete(ifobj_tx->xsk->xsk);
1422 xsk_socket__delete(ifobj_rx->xsk->xsk);
1423 testapp_clean_xsk_umem(ifobj_rx);
1424 if (!ifobj_tx->shared_umem)
1425 testapp_clean_xsk_umem(ifobj_tx);
1428 return !!test->fail;
1431 static void testapp_teardown(struct test_spec *test)
1435 test_spec_set_name(test, "TEARDOWN");
1436 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1437 if (testapp_validate_traffic(test))
1439 test_spec_reset(test);
1443 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1445 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1446 struct ifobject *tmp_ifobj = (*ifobj1);
1448 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1449 (*ifobj2)->func_ptr = tmp_func_ptr;
1452 *ifobj2 = tmp_ifobj;
1455 static void testapp_bidi(struct test_spec *test)
1457 test_spec_set_name(test, "BIDIRECTIONAL");
1458 test->ifobj_tx->rx_on = true;
1459 test->ifobj_rx->tx_on = true;
1460 test->total_steps = 2;
1461 if (testapp_validate_traffic(test))
1464 print_verbose("Switching Tx/Rx vectors\n");
1465 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1466 testapp_validate_traffic(test);
1468 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1471 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
1475 xsk_socket__delete(ifobj_tx->xsk->xsk);
1476 xsk_socket__delete(ifobj_rx->xsk->xsk);
1477 ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
1478 ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
1480 ret = xsk_socket__update_xskmap(ifobj_rx->xsk->xsk, ifobj_rx->xsk_map_fd);
1482 exit_with_error(-ret);
1485 static void testapp_bpf_res(struct test_spec *test)
1487 test_spec_set_name(test, "BPF_RES");
1488 test->total_steps = 2;
1489 test->nb_sockets = 2;
1490 if (testapp_validate_traffic(test))
1493 swap_xsk_resources(test->ifobj_tx, test->ifobj_rx);
1494 testapp_validate_traffic(test);
1497 static void testapp_headroom(struct test_spec *test)
1499 test_spec_set_name(test, "UMEM_HEADROOM");
1500 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1501 testapp_validate_traffic(test);
1504 static void testapp_stats_rx_dropped(struct test_spec *test)
1506 test_spec_set_name(test, "STAT_RX_DROPPED");
1507 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1508 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1509 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1510 pkt_stream_receive_half(test);
1511 test->ifobj_rx->validation_func = validate_rx_dropped;
1512 testapp_validate_traffic(test);
1515 static void testapp_stats_tx_invalid_descs(struct test_spec *test)
1517 test_spec_set_name(test, "STAT_TX_INVALID");
1518 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
1519 test->ifobj_tx->validation_func = validate_tx_invalid_descs;
1520 testapp_validate_traffic(test);
1522 pkt_stream_restore_default(test);
1525 static void testapp_stats_rx_full(struct test_spec *test)
1527 test_spec_set_name(test, "STAT_RX_FULL");
1528 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1529 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1530 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1531 if (!test->ifobj_rx->pkt_stream)
1532 exit_with_error(ENOMEM);
1534 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
1535 test->ifobj_rx->release_rx = false;
1536 test->ifobj_rx->validation_func = validate_rx_full;
1537 testapp_validate_traffic(test);
1539 pkt_stream_restore_default(test);
1542 static void testapp_stats_fill_empty(struct test_spec *test)
1544 test_spec_set_name(test, "STAT_RX_FILL_EMPTY");
1545 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1546 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1547 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1548 if (!test->ifobj_rx->pkt_stream)
1549 exit_with_error(ENOMEM);
1551 test->ifobj_rx->use_fill_ring = false;
1552 test->ifobj_rx->validation_func = validate_fill_empty;
1553 testapp_validate_traffic(test);
1555 pkt_stream_restore_default(test);
1559 static bool hugepages_present(struct ifobject *ifobject)
1561 const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size;
1564 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1565 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
1566 if (bufs == MAP_FAILED)
1569 munmap(bufs, mmap_sz);
1573 static bool testapp_unaligned(struct test_spec *test)
1575 if (!hugepages_present(test->ifobj_tx)) {
1576 ksft_test_result_skip("No 2M huge pages present.\n");
1580 test_spec_set_name(test, "UNALIGNED_MODE");
1581 test->ifobj_tx->umem->unaligned_mode = true;
1582 test->ifobj_rx->umem->unaligned_mode = true;
1583 /* Let half of the packets straddle a buffer boundrary */
1584 pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2);
1585 test->ifobj_rx->pkt_stream->use_addr_for_fill = true;
1586 testapp_validate_traffic(test);
1588 pkt_stream_restore_default(test);
1592 static void testapp_single_pkt(struct test_spec *test)
1594 struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}};
1596 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1597 testapp_validate_traffic(test);
1598 pkt_stream_restore_default(test);
1601 static void testapp_invalid_desc(struct test_spec *test)
1603 struct pkt pkts[] = {
1604 /* Zero packet address allowed */
1605 {0, PKT_SIZE, 0, true},
1606 /* Allowed packet */
1607 {0x1000, PKT_SIZE, 0, true},
1608 /* Straddling the start of umem */
1609 {-2, PKT_SIZE, 0, false},
1610 /* Packet too large */
1611 {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
1612 /* After umem ends */
1613 {UMEM_SIZE, PKT_SIZE, 0, false},
1614 /* Straddle the end of umem */
1615 {UMEM_SIZE - PKT_SIZE / 2, PKT_SIZE, 0, false},
1616 /* Straddle a page boundrary */
1617 {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false},
1618 /* Straddle a 2K boundrary */
1619 {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true},
1620 /* Valid packet for synch so that something is received */
1621 {0x4000, PKT_SIZE, 0, true}};
1623 if (test->ifobj_tx->umem->unaligned_mode) {
1624 /* Crossing a page boundrary allowed */
1625 pkts[6].valid = true;
1627 if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
1628 /* Crossing a 2K frame size boundrary not allowed */
1629 pkts[7].valid = false;
1632 if (test->ifobj_tx->shared_umem) {
1633 pkts[4].addr += UMEM_SIZE;
1634 pkts[5].addr += UMEM_SIZE;
1637 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1638 testapp_validate_traffic(test);
1639 pkt_stream_restore_default(test);
1642 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
1643 const char *dst_ip, const char *src_ip, const u16 dst_port,
1644 const u16 src_port, thread_func_t func_ptr)
1648 memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
1649 memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
1651 inet_aton(dst_ip, &ip);
1652 ifobj->dst_ip = ip.s_addr;
1654 inet_aton(src_ip, &ip);
1655 ifobj->src_ip = ip.s_addr;
1657 ifobj->dst_port = dst_port;
1658 ifobj->src_port = src_port;
1660 ifobj->func_ptr = func_ptr;
1663 static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type)
1666 case TEST_TYPE_STATS_RX_DROPPED:
1667 testapp_stats_rx_dropped(test);
1669 case TEST_TYPE_STATS_TX_INVALID_DESCS:
1670 testapp_stats_tx_invalid_descs(test);
1672 case TEST_TYPE_STATS_RX_FULL:
1673 testapp_stats_rx_full(test);
1675 case TEST_TYPE_STATS_FILL_EMPTY:
1676 testapp_stats_fill_empty(test);
1678 case TEST_TYPE_TEARDOWN:
1679 testapp_teardown(test);
1681 case TEST_TYPE_BIDI:
1684 case TEST_TYPE_BPF_RES:
1685 testapp_bpf_res(test);
1687 case TEST_TYPE_RUN_TO_COMPLETION:
1688 test_spec_set_name(test, "RUN_TO_COMPLETION");
1689 testapp_validate_traffic(test);
1691 case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT:
1692 test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT");
1693 testapp_single_pkt(test);
1695 case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME:
1696 test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE");
1697 test->ifobj_tx->umem->frame_size = 2048;
1698 test->ifobj_rx->umem->frame_size = 2048;
1699 pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE);
1700 testapp_validate_traffic(test);
1702 pkt_stream_restore_default(test);
1704 case TEST_TYPE_RX_POLL:
1705 test->ifobj_rx->use_poll = true;
1706 test_spec_set_name(test, "POLL_RX");
1707 testapp_validate_traffic(test);
1709 case TEST_TYPE_TX_POLL:
1710 test->ifobj_tx->use_poll = true;
1711 test_spec_set_name(test, "POLL_TX");
1712 testapp_validate_traffic(test);
1714 case TEST_TYPE_POLL_TXQ_TMOUT:
1715 test_spec_set_name(test, "POLL_TXQ_FULL");
1716 test->ifobj_tx->use_poll = true;
1717 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */
1718 test->ifobj_tx->umem->frame_size = 2048;
1719 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
1720 testapp_validate_traffic_single_thread(test, test->ifobj_tx, type);
1721 pkt_stream_restore_default(test);
1723 case TEST_TYPE_POLL_RXQ_TMOUT:
1724 test_spec_set_name(test, "POLL_RXQ_EMPTY");
1725 test->ifobj_rx->use_poll = true;
1726 testapp_validate_traffic_single_thread(test, test->ifobj_rx, type);
1728 case TEST_TYPE_ALIGNED_INV_DESC:
1729 test_spec_set_name(test, "ALIGNED_INV_DESC");
1730 testapp_invalid_desc(test);
1732 case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME:
1733 test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE");
1734 test->ifobj_tx->umem->frame_size = 2048;
1735 test->ifobj_rx->umem->frame_size = 2048;
1736 testapp_invalid_desc(test);
1738 case TEST_TYPE_UNALIGNED_INV_DESC:
1739 if (!hugepages_present(test->ifobj_tx)) {
1740 ksft_test_result_skip("No 2M huge pages present.\n");
1743 test_spec_set_name(test, "UNALIGNED_INV_DESC");
1744 test->ifobj_tx->umem->unaligned_mode = true;
1745 test->ifobj_rx->umem->unaligned_mode = true;
1746 testapp_invalid_desc(test);
1748 case TEST_TYPE_UNALIGNED:
1749 if (!testapp_unaligned(test))
1752 case TEST_TYPE_HEADROOM:
1753 testapp_headroom(test);
1760 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
1764 static struct ifobject *ifobject_create(void)
1766 struct ifobject *ifobj;
1768 ifobj = calloc(1, sizeof(struct ifobject));
1772 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
1773 if (!ifobj->xsk_arr)
1776 ifobj->umem = calloc(1, sizeof(*ifobj->umem));
1783 free(ifobj->xsk_arr);
1789 static void ifobject_delete(struct ifobject *ifobj)
1792 free(ifobj->xsk_arr);
1796 static bool is_xdp_supported(struct ifobject *ifobject)
1798 int flags = XDP_FLAGS_DRV_MODE;
1800 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
1801 struct bpf_insn insns[2] = {
1802 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
1805 int ifindex = if_nametoindex(ifobject->ifname);
1806 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
1809 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
1813 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
1819 bpf_xdp_detach(ifindex, flags, NULL);
1825 int main(int argc, char **argv)
1827 struct pkt_stream *rx_pkt_stream_default;
1828 struct pkt_stream *tx_pkt_stream_default;
1829 struct ifobject *ifobj_tx, *ifobj_rx;
1830 int modes = TEST_MODE_SKB + 1;
1831 u32 i, j, failed_tests = 0;
1832 struct test_spec test;
1835 /* Use libbpf 1.0 API mode */
1836 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1838 ifobj_tx = ifobject_create();
1840 exit_with_error(ENOMEM);
1841 ifobj_rx = ifobject_create();
1843 exit_with_error(ENOMEM);
1845 setlocale(LC_ALL, "");
1847 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
1848 shared_umem = !strcmp(ifobj_tx->ifname, ifobj_rx->ifname);
1850 ifobj_tx->shared_umem = shared_umem;
1851 ifobj_rx->shared_umem = shared_umem;
1853 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
1854 usage(basename(argv[0]));
1858 init_iface(ifobj_tx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2,
1859 worker_testapp_validate_tx);
1860 init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
1861 worker_testapp_validate_rx);
1863 if (is_xdp_supported(ifobj_tx))
1866 test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
1867 tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1868 rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1869 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
1870 exit_with_error(ENOMEM);
1871 test.tx_pkt_stream_default = tx_pkt_stream_default;
1872 test.rx_pkt_stream_default = rx_pkt_stream_default;
1874 ksft_set_plan(modes * TEST_TYPE_MAX);
1876 for (i = 0; i < modes; i++)
1877 for (j = 0; j < TEST_TYPE_MAX; j++) {
1878 test_spec_init(&test, ifobj_tx, ifobj_rx, i);
1879 run_pkt_test(&test, i, j);
1886 pkt_stream_delete(tx_pkt_stream_default);
1887 pkt_stream_delete(rx_pkt_stream_default);
1888 free(ifobj_rx->umem);
1889 if (!ifobj_tx->shared_umem)
1890 free(ifobj_tx->umem);
1891 ifobject_delete(ifobj_tx);
1892 ifobject_delete(ifobj_rx);