tools/testing/selftests/bpf/xdp_hw_metadata.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /* Reference program for verifying XDP metadata on real HW. Functional test
   4  * only, doesn't test the performance.
   5  *
   6  * RX:
   7  * - UDP 9091 packets are diverted into AF_XDP
   8  * - Metadata verified:
   9  *   - rx_timestamp
  10  *   - rx_hash
  11  *
  12  * TX:
  13  * - TBD
  14  */
  15
  16 #include <test_progs.h>
  17 #include <network_helpers.h>
  18 #include "xdp_hw_metadata.skel.h"
  19 #include "xsk.h"
  20
  21 #include <error.h>
  22 #include <linux/errqueue.h>
  23 #include <linux/if_link.h>
  24 #include <linux/net_tstamp.h>
  25 #include <linux/udp.h>
  26 #include <linux/sockios.h>
  27 #include <sys/mman.h>
  28 #include <net/if.h>
  29 #include <poll.h>
  30
  31 #include "xdp_metadata.h"
  32
  33 #define UMEM_NUM 16
  34 #define UMEM_FRAME_SIZE XSK_UMEM__DEFAULT_FRAME_SIZE
  35 #define UMEM_SIZE (UMEM_FRAME_SIZE * UMEM_NUM)
  36 #define XDP_FLAGS (XDP_FLAGS_DRV_MODE | XDP_FLAGS_REPLACE)
  37
  38 struct xsk {
  39         void *umem_area;
  40         struct xsk_umem *umem;
  41         struct xsk_ring_prod fill;
  42         struct xsk_ring_cons comp;
  43         struct xsk_ring_prod tx;
  44         struct xsk_ring_cons rx;
  45         struct xsk_socket *socket;
  46 };
  47
  48 struct xdp_hw_metadata *bpf_obj;
  49 struct xsk *rx_xsk;
  50 const char *ifname;
  51 int ifindex;
  52 int rxq;
  53
  54 void test__fail(void) { /* for network_helpers.c */ }
  55
  56 static int open_xsk(int ifindex, struct xsk *xsk, __u32 queue_id)
  57 {
  58         int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
  59         const struct xsk_socket_config socket_config = {
  60                 .rx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
  61                 .tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
  62                 .bind_flags = XDP_COPY,
  63         };
  64         const struct xsk_umem_config umem_config = {
  65                 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
  66                 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
  67                 .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE,
  68                 .flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG,
  69         };
  70         __u32 idx;
  71         u64 addr;
  72         int ret;
  73         int i;
  74
  75         xsk->umem_area = mmap(NULL, UMEM_SIZE, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
  76         if (xsk->umem_area == MAP_FAILED)
  77                 return -ENOMEM;
  78
  79         ret = xsk_umem__create(&xsk->umem,
  80                                xsk->umem_area, UMEM_SIZE,
  81                                &xsk->fill,
  82                                &xsk->comp,
  83                                &umem_config);
  84         if (ret)
  85                 return ret;
  86
  87         ret = xsk_socket__create(&xsk->socket, ifindex, queue_id,
  88                                  xsk->umem,
  89                                  &xsk->rx,
  90                                  &xsk->tx,
  91                                  &socket_config);
  92         if (ret)
  93                 return ret;
  94
  95         /* First half of umem is for TX. This way address matches 1-to-1
  96          * to the completion queue index.
  97          */
  98
  99         for (i = 0; i < UMEM_NUM / 2; i++) {
 100                 addr = i * UMEM_FRAME_SIZE;
 101                 printf("%p: tx_desc[%d] -> %lx\n", xsk, i, addr);
 102         }
 103
 104         /* Second half of umem is for RX. */
 105
 106         ret = xsk_ring_prod__reserve(&xsk->fill, UMEM_NUM / 2, &idx);
 107         for (i = 0; i < UMEM_NUM / 2; i++) {
 108                 addr = (UMEM_NUM / 2 + i) * UMEM_FRAME_SIZE;
 109                 printf("%p: rx_desc[%d] -> %lx\n", xsk, i, addr);
 110                 *xsk_ring_prod__fill_addr(&xsk->fill, i) = addr;
 111         }
 112         xsk_ring_prod__submit(&xsk->fill, ret);
 113
 114         return 0;
 115 }
 116
 117 static void close_xsk(struct xsk *xsk)
 118 {
 119         if (xsk->umem)
 120                 xsk_umem__delete(xsk->umem);
 121         if (xsk->socket)
 122                 xsk_socket__delete(xsk->socket);
 123         munmap(xsk->umem_area, UMEM_SIZE);
 124 }
 125
 126 static void refill_rx(struct xsk *xsk, __u64 addr)
 127 {
 128         __u32 idx;
 129
 130         if (xsk_ring_prod__reserve(&xsk->fill, 1, &idx) == 1) {
 131                 printf("%p: complete idx=%u addr=%llx\n", xsk, idx, addr);
 132                 *xsk_ring_prod__fill_addr(&xsk->fill, idx) = addr;
 133                 xsk_ring_prod__submit(&xsk->fill, 1);
 134         }
 135 }
 136
 137 static void verify_xdp_metadata(void *data)
 138 {
 139         struct xdp_meta *meta;
 140
 141         meta = data - sizeof(*meta);
 142
 143         printf("rx_timestamp: %llu\n", meta->rx_timestamp);
 144         printf("rx_hash: %u\n", meta->rx_hash);
 145 }
 146
 147 static void verify_skb_metadata(int fd)
 148 {
 149         char cmsg_buf[1024];
 150         char packet_buf[128];
 151
 152         struct scm_timestamping *ts;
 153         struct iovec packet_iov;
 154         struct cmsghdr *cmsg;
 155         struct msghdr hdr;
 156
 157         memset(&hdr, 0, sizeof(hdr));
 158         hdr.msg_iov = &packet_iov;
 159         hdr.msg_iovlen = 1;
 160         packet_iov.iov_base = packet_buf;
 161         packet_iov.iov_len = sizeof(packet_buf);
 162
 163         hdr.msg_control = cmsg_buf;
 164         hdr.msg_controllen = sizeof(cmsg_buf);
 165
 166         if (recvmsg(fd, &hdr, 0) < 0)
 167                 error(1, errno, "recvmsg");
 168
 169         for (cmsg = CMSG_FIRSTHDR(&hdr); cmsg != NULL;
 170              cmsg = CMSG_NXTHDR(&hdr, cmsg)) {
 171
 172                 if (cmsg->cmsg_level != SOL_SOCKET)
 173                         continue;
 174
 175                 switch (cmsg->cmsg_type) {
 176                 case SCM_TIMESTAMPING:
 177                         ts = (struct scm_timestamping *)CMSG_DATA(cmsg);
 178                         if (ts->ts[2].tv_sec || ts->ts[2].tv_nsec) {
 179                                 printf("found skb hwtstamp = %lu.%lu\n",
 180                                        ts->ts[2].tv_sec, ts->ts[2].tv_nsec);
 181                                 return;
 182                         }
 183                         break;
 184                 default:
 185                         break;
 186                 }
 187         }
 188
 189         printf("skb hwtstamp is not found!\n");
 190 }
 191
 192 static int verify_metadata(struct xsk *rx_xsk, int rxq, int server_fd)
 193 {
 194         const struct xdp_desc *rx_desc;
 195         struct pollfd fds[rxq + 1];
 196         __u64 comp_addr;
 197         __u64 addr;
 198         __u32 idx;
 199         int ret;
 200         int i;
 201
 202         for (i = 0; i < rxq; i++) {
 203                 fds[i].fd = xsk_socket__fd(rx_xsk[i].socket);
 204                 fds[i].events = POLLIN;
 205                 fds[i].revents = 0;
 206         }
 207
 208         fds[rxq].fd = server_fd;
 209         fds[rxq].events = POLLIN;
 210         fds[rxq].revents = 0;
 211
 212         while (true) {
 213                 errno = 0;
 214                 ret = poll(fds, rxq + 1, 1000);
 215                 printf("poll: %d (%d) skip=%llu fail=%llu redir=%llu\n",
 216                        ret, errno, bpf_obj->bss->pkts_skip,
 217                        bpf_obj->bss->pkts_fail, bpf_obj->bss->pkts_redir);
 218                 if (ret < 0)
 219                         break;
 220                 if (ret == 0)
 221                         continue;
 222
 223                 if (fds[rxq].revents)
 224                         verify_skb_metadata(server_fd);
 225
 226                 for (i = 0; i < rxq; i++) {
 227                         if (fds[i].revents == 0)
 228                                 continue;
 229
 230                         struct xsk *xsk = &rx_xsk[i];
 231
 232                         ret = xsk_ring_cons__peek(&xsk->rx, 1, &idx);
 233                         printf("xsk_ring_cons__peek: %d\n", ret);
 234                         if (ret != 1)
 235                                 continue;
 236
 237                         rx_desc = xsk_ring_cons__rx_desc(&xsk->rx, idx);
 238                         comp_addr = xsk_umem__extract_addr(rx_desc->addr);
 239                         addr = xsk_umem__add_offset_to_addr(rx_desc->addr);
 240                         printf("%p: rx_desc[%u]->addr=%llx addr=%llx comp_addr=%llx\n",
 241                                xsk, idx, rx_desc->addr, addr, comp_addr);
 242                         verify_xdp_metadata(xsk_umem__get_data(xsk->umem_area, addr));
 243                         xsk_ring_cons__release(&xsk->rx, 1);
 244                         refill_rx(xsk, comp_addr);
 245                 }
 246         }
 247
 248         return 0;
 249 }
 250
 251 struct ethtool_channels {
 252         __u32   cmd;
 253         __u32   max_rx;
 254         __u32   max_tx;
 255         __u32   max_other;
 256         __u32   max_combined;
 257         __u32   rx_count;
 258         __u32   tx_count;
 259         __u32   other_count;
 260         __u32   combined_count;
 261 };
 262
 263 #define ETHTOOL_GCHANNELS       0x0000003c /* Get no of channels */
 264
 265 static int rxq_num(const char *ifname)
 266 {
 267         struct ethtool_channels ch = {
 268                 .cmd = ETHTOOL_GCHANNELS,
 269         };
 270
 271         struct ifreq ifr = {
 272                 .ifr_data = (void *)&ch,
 273         };
 274         strncpy(ifr.ifr_name, ifname, IF_NAMESIZE - 1);
 275         int fd, ret;
 276
 277         fd = socket(AF_UNIX, SOCK_DGRAM, 0);
 278         if (fd < 0)
 279                 error(1, errno, "socket");
 280
 281         ret = ioctl(fd, SIOCETHTOOL, &ifr);
 282         if (ret < 0)
 283                 error(1, errno, "ioctl(SIOCETHTOOL)");
 284
 285         close(fd);
 286
 287         return ch.rx_count + ch.combined_count;
 288 }
 289
 290 static void hwtstamp_ioctl(int op, const char *ifname, struct hwtstamp_config *cfg)
 291 {
 292         struct ifreq ifr = {
 293                 .ifr_data = (void *)cfg,
 294         };
 295         strncpy(ifr.ifr_name, ifname, IF_NAMESIZE - 1);
 296         int fd, ret;
 297
 298         fd = socket(AF_UNIX, SOCK_DGRAM, 0);
 299         if (fd < 0)
 300                 error(1, errno, "socket");
 301
 302         ret = ioctl(fd, op, &ifr);
 303         if (ret < 0)
 304                 error(1, errno, "ioctl(%d)", op);
 305
 306         close(fd);
 307 }
 308
 309 static struct hwtstamp_config saved_hwtstamp_cfg;
 310 static const char *saved_hwtstamp_ifname;
 311
 312 static void hwtstamp_restore(void)
 313 {
 314         hwtstamp_ioctl(SIOCSHWTSTAMP, saved_hwtstamp_ifname, &saved_hwtstamp_cfg);
 315 }
 316
 317 static void hwtstamp_enable(const char *ifname)
 318 {
 319         struct hwtstamp_config cfg = {
 320                 .rx_filter = HWTSTAMP_FILTER_ALL,
 321         };
 322
 323         hwtstamp_ioctl(SIOCGHWTSTAMP, ifname, &saved_hwtstamp_cfg);
 324         saved_hwtstamp_ifname = strdup(ifname);
 325         atexit(hwtstamp_restore);
 326
 327         hwtstamp_ioctl(SIOCSHWTSTAMP, ifname, &cfg);
 328 }
 329
 330 static void cleanup(void)
 331 {
 332         LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
 333         int ret;
 334         int i;
 335
 336         if (bpf_obj) {
 337                 opts.old_prog_fd = bpf_program__fd(bpf_obj->progs.rx);
 338                 if (opts.old_prog_fd >= 0) {
 339                         printf("detaching bpf program....\n");
 340                         ret = bpf_xdp_detach(ifindex, XDP_FLAGS, &opts);
 341                         if (ret)
 342                                 printf("failed to detach XDP program: %d\n", ret);
 343                 }
 344         }
 345
 346         for (i = 0; i < rxq; i++)
 347                 close_xsk(&rx_xsk[i]);
 348
 349         if (bpf_obj)
 350                 xdp_hw_metadata__destroy(bpf_obj);
 351 }
 352
 353 static void handle_signal(int sig)
 354 {
 355         /* interrupting poll() is all we need */
 356 }
 357
 358 static void timestamping_enable(int fd, int val)
 359 {
 360         int ret;
 361
 362         ret = setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val));
 363         if (ret < 0)
 364                 error(1, errno, "setsockopt(SO_TIMESTAMPING)");
 365 }
 366
 367 int main(int argc, char *argv[])
 368 {
 369         int server_fd = -1;
 370         int ret;
 371         int i;
 372
 373         struct bpf_program *prog;
 374
 375         if (argc != 2) {
 376                 fprintf(stderr, "pass device name\n");
 377                 return -1;
 378         }
 379
 380         ifname = argv[1];
 381         ifindex = if_nametoindex(ifname);
 382         rxq = rxq_num(ifname);
 383
 384         printf("rxq: %d\n", rxq);
 385
 386         hwtstamp_enable(ifname);
 387
 388         rx_xsk = malloc(sizeof(struct xsk) * rxq);
 389         if (!rx_xsk)
 390                 error(1, ENOMEM, "malloc");
 391
 392         for (i = 0; i < rxq; i++) {
 393                 printf("open_xsk(%s, %p, %d)\n", ifname, &rx_xsk[i], i);
 394                 ret = open_xsk(ifindex, &rx_xsk[i], i);
 395                 if (ret)
 396                         error(1, -ret, "open_xsk");
 397
 398                 printf("xsk_socket__fd() -> %d\n", xsk_socket__fd(rx_xsk[i].socket));
 399         }
 400
 401         printf("open bpf program...\n");
 402         bpf_obj = xdp_hw_metadata__open();
 403         if (libbpf_get_error(bpf_obj))
 404                 error(1, libbpf_get_error(bpf_obj), "xdp_hw_metadata__open");
 405
 406         prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx");
 407         bpf_program__set_ifindex(prog, ifindex);
 408         bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY);
 409
 410         printf("load bpf program...\n");
 411         ret = xdp_hw_metadata__load(bpf_obj);
 412         if (ret)
 413                 error(1, -ret, "xdp_hw_metadata__load");
 414
 415         printf("prepare skb endpoint...\n");
 416         server_fd = start_server(AF_INET6, SOCK_DGRAM, NULL, 9092, 1000);
 417         if (server_fd < 0)
 418                 error(1, errno, "start_server");
 419         timestamping_enable(server_fd,
 420                             SOF_TIMESTAMPING_SOFTWARE |
 421                             SOF_TIMESTAMPING_RAW_HARDWARE);
 422
 423         printf("prepare xsk map...\n");
 424         for (i = 0; i < rxq; i++) {
 425                 int sock_fd = xsk_socket__fd(rx_xsk[i].socket);
 426                 __u32 queue_id = i;
 427
 428                 printf("map[%d] = %d\n", queue_id, sock_fd);
 429                 ret = bpf_map_update_elem(bpf_map__fd(bpf_obj->maps.xsk), &queue_id, &sock_fd, 0);
 430                 if (ret)
 431                         error(1, -ret, "bpf_map_update_elem");
 432         }
 433
 434         printf("attach bpf program...\n");
 435         ret = bpf_xdp_attach(ifindex,
 436                              bpf_program__fd(bpf_obj->progs.rx),
 437                              XDP_FLAGS, NULL);
 438         if (ret)
 439                 error(1, -ret, "bpf_xdp_attach");
 440
 441         signal(SIGINT, handle_signal);
 442         ret = verify_metadata(rx_xsk, rxq, server_fd);
 443         close(server_fd);
 444         cleanup();
 445         if (ret)
 446                 error(1, -ret, "verify_metadata");
 447 }