net/xdp/xsk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* XDP sockets
   3  *
   4  * AF_XDP sockets allows a channel between XDP programs and userspace
   5  * applications.
   6  * Copyright(c) 2018 Intel Corporation.
   7  *
   8  * Author(s): Björn Töpel <bjorn.topel@intel.com>
   9  *            Magnus Karlsson <magnus.karlsson@intel.com>
  10  */
  11
  12 #define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
  13
  14 #include <linux/if_xdp.h>
  15 #include <linux/init.h>
  16 #include <linux/sched/mm.h>
  17 #include <linux/sched/signal.h>
  18 #include <linux/sched/task.h>
  19 #include <linux/socket.h>
  20 #include <linux/file.h>
  21 #include <linux/uaccess.h>
  22 #include <linux/net.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/rculist.h>
  25 #include <linux/vmalloc.h>
  26 #include <net/xdp_sock_drv.h>
  27 #include <net/busy_poll.h>
  28 #include <net/netdev_rx_queue.h>
  29 #include <net/xdp.h>
  30
  31 #include "xsk_queue.h"
  32 #include "xdp_umem.h"
  33 #include "xsk.h"
  34
  35 #define TX_BATCH_SIZE 32
  36
  37 static DEFINE_PER_CPU(struct list_head, xskmap_flush_list);
  38
  39 void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
  40 {
  41         if (pool->cached_need_wakeup & XDP_WAKEUP_RX)
  42                 return;
  43
  44         pool->fq->ring->flags |= XDP_RING_NEED_WAKEUP;
  45         pool->cached_need_wakeup |= XDP_WAKEUP_RX;
  46 }
  47 EXPORT_SYMBOL(xsk_set_rx_need_wakeup);
  48
  49 void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
  50 {
  51         struct xdp_sock *xs;
  52
  53         if (pool->cached_need_wakeup & XDP_WAKEUP_TX)
  54                 return;
  55
  56         rcu_read_lock();
  57         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
  58                 xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
  59         }
  60         rcu_read_unlock();
  61
  62         pool->cached_need_wakeup |= XDP_WAKEUP_TX;
  63 }
  64 EXPORT_SYMBOL(xsk_set_tx_need_wakeup);
  65
  66 void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
  67 {
  68         if (!(pool->cached_need_wakeup & XDP_WAKEUP_RX))
  69                 return;
  70
  71         pool->fq->ring->flags &= ~XDP_RING_NEED_WAKEUP;
  72         pool->cached_need_wakeup &= ~XDP_WAKEUP_RX;
  73 }
  74 EXPORT_SYMBOL(xsk_clear_rx_need_wakeup);
  75
  76 void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
  77 {
  78         struct xdp_sock *xs;
  79
  80         if (!(pool->cached_need_wakeup & XDP_WAKEUP_TX))
  81                 return;
  82
  83         rcu_read_lock();
  84         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
  85                 xs->tx->ring->flags &= ~XDP_RING_NEED_WAKEUP;
  86         }
  87         rcu_read_unlock();
  88
  89         pool->cached_need_wakeup &= ~XDP_WAKEUP_TX;
  90 }
  91 EXPORT_SYMBOL(xsk_clear_tx_need_wakeup);
  92
  93 bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
  94 {
  95         return pool->uses_need_wakeup;
  96 }
  97 EXPORT_SYMBOL(xsk_uses_need_wakeup);
  98
  99 struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
 100                                             u16 queue_id)
 101 {
 102         if (queue_id < dev->real_num_rx_queues)
 103                 return dev->_rx[queue_id].pool;
 104         if (queue_id < dev->real_num_tx_queues)
 105                 return dev->_tx[queue_id].pool;
 106
 107         return NULL;
 108 }
 109 EXPORT_SYMBOL(xsk_get_pool_from_qid);
 110
 111 void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id)
 112 {
 113         if (queue_id < dev->num_rx_queues)
 114                 dev->_rx[queue_id].pool = NULL;
 115         if (queue_id < dev->num_tx_queues)
 116                 dev->_tx[queue_id].pool = NULL;
 117 }
 118
 119 /* The buffer pool is stored both in the _rx struct and the _tx struct as we do
 120  * not know if the device has more tx queues than rx, or the opposite.
 121  * This might also change during run time.
 122  */
 123 int xsk_reg_pool_at_qid(struct net_device *dev, struct xsk_buff_pool *pool,
 124                         u16 queue_id)
 125 {
 126         if (queue_id >= max_t(unsigned int,
 127                               dev->real_num_rx_queues,
 128                               dev->real_num_tx_queues))
 129                 return -EINVAL;
 130
 131         if (queue_id < dev->real_num_rx_queues)
 132                 dev->_rx[queue_id].pool = pool;
 133         if (queue_id < dev->real_num_tx_queues)
 134                 dev->_tx[queue_id].pool = pool;
 135
 136         return 0;
 137 }
 138
 139 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len,
 140                         u32 flags)
 141 {
 142         u64 addr;
 143         int err;
 144
 145         addr = xp_get_handle(xskb);
 146         err = xskq_prod_reserve_desc(xs->rx, addr, len, flags);
 147         if (err) {
 148                 xs->rx_queue_full++;
 149                 return err;
 150         }
 151
 152         xp_release(xskb);
 153         return 0;
 154 }
 155
 156 static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 157 {
 158         struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
 159         u32 frags = xdp_buff_has_frags(xdp);
 160         struct xdp_buff_xsk *pos, *tmp;
 161         struct list_head *xskb_list;
 162         u32 contd = 0;
 163         int err;
 164
 165         if (frags)
 166                 contd = XDP_PKT_CONTD;
 167
 168         err = __xsk_rcv_zc(xs, xskb, len, contd);
 169         if (err || likely(!frags))
 170                 goto out;
 171
 172         xskb_list = &xskb->pool->xskb_list;
 173         list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) {
 174                 if (list_is_singular(xskb_list))
 175                         contd = 0;
 176                 len = pos->xdp.data_end - pos->xdp.data;
 177                 err = __xsk_rcv_zc(xs, pos, len, contd);
 178                 if (err)
 179                         return err;
 180                 list_del(&pos->xskb_list_node);
 181         }
 182
 183 out:
 184         return err;
 185 }
 186
 187 static void *xsk_copy_xdp_start(struct xdp_buff *from)
 188 {
 189         if (unlikely(xdp_data_meta_unsupported(from)))
 190                 return from->data;
 191         else
 192                 return from->data_meta;
 193 }
 194
 195 static u32 xsk_copy_xdp(void *to, void **from, u32 to_len,
 196                         u32 *from_len, skb_frag_t **frag, u32 rem)
 197 {
 198         u32 copied = 0;
 199
 200         while (1) {
 201                 u32 copy_len = min_t(u32, *from_len, to_len);
 202
 203                 memcpy(to, *from, copy_len);
 204                 copied += copy_len;
 205                 if (rem == copied)
 206                         return copied;
 207
 208                 if (*from_len == copy_len) {
 209                         *from = skb_frag_address(*frag);
 210                         *from_len = skb_frag_size((*frag)++);
 211                 } else {
 212                         *from += copy_len;
 213                         *from_len -= copy_len;
 214                 }
 215                 if (to_len == copy_len)
 216                         return copied;
 217
 218                 to_len -= copy_len;
 219                 to += copy_len;
 220         }
 221 }
 222
 223 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 224 {
 225         u32 frame_size = xsk_pool_get_rx_frame_size(xs->pool);
 226         void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
 227         u32 from_len, meta_len, rem, num_desc;
 228         struct xdp_buff_xsk *xskb;
 229         struct xdp_buff *xsk_xdp;
 230         skb_frag_t *frag;
 231
 232         from_len = xdp->data_end - copy_from;
 233         meta_len = xdp->data - copy_from;
 234         rem = len + meta_len;
 235
 236         if (len <= frame_size && !xdp_buff_has_frags(xdp)) {
 237                 int err;
 238
 239                 xsk_xdp = xsk_buff_alloc(xs->pool);
 240                 if (!xsk_xdp) {
 241                         xs->rx_dropped++;
 242                         return -ENOMEM;
 243                 }
 244                 memcpy(xsk_xdp->data - meta_len, copy_from, rem);
 245                 xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 246                 err = __xsk_rcv_zc(xs, xskb, len, 0);
 247                 if (err) {
 248                         xsk_buff_free(xsk_xdp);
 249                         return err;
 250                 }
 251
 252                 return 0;
 253         }
 254
 255         num_desc = (len - 1) / frame_size + 1;
 256
 257         if (!xsk_buff_can_alloc(xs->pool, num_desc)) {
 258                 xs->rx_dropped++;
 259                 return -ENOMEM;
 260         }
 261         if (xskq_prod_nb_free(xs->rx, num_desc) < num_desc) {
 262                 xs->rx_queue_full++;
 263                 return -ENOBUFS;
 264         }
 265
 266         if (xdp_buff_has_frags(xdp)) {
 267                 struct skb_shared_info *sinfo;
 268
 269                 sinfo = xdp_get_shared_info_from_buff(xdp);
 270                 frag =  &sinfo->frags[0];
 271         }
 272
 273         do {
 274                 u32 to_len = frame_size + meta_len;
 275                 u32 copied;
 276
 277                 xsk_xdp = xsk_buff_alloc(xs->pool);
 278                 copy_to = xsk_xdp->data - meta_len;
 279
 280                 copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
 281                 rem -= copied;
 282
 283                 xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 284                 __xsk_rcv_zc(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0);
 285                 meta_len = 0;
 286         } while (rem);
 287
 288         return 0;
 289 }
 290
 291 static bool xsk_tx_writeable(struct xdp_sock *xs)
 292 {
 293         if (xskq_cons_present_entries(xs->tx) > xs->tx->nentries / 2)
 294                 return false;
 295
 296         return true;
 297 }
 298
 299 static bool xsk_is_bound(struct xdp_sock *xs)
 300 {
 301         if (READ_ONCE(xs->state) == XSK_BOUND) {
 302                 /* Matches smp_wmb() in bind(). */
 303                 smp_rmb();
 304                 return true;
 305         }
 306         return false;
 307 }
 308
 309 static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 310 {
 311         if (!xsk_is_bound(xs))
 312                 return -ENXIO;
 313
 314         if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
 315                 return -EINVAL;
 316
 317         if (len > xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) {
 318                 xs->rx_dropped++;
 319                 return -ENOSPC;
 320         }
 321
 322         sk_mark_napi_id_once_xdp(&xs->sk, xdp);
 323         return 0;
 324 }
 325
 326 static void xsk_flush(struct xdp_sock *xs)
 327 {
 328         xskq_prod_submit(xs->rx);
 329         __xskq_cons_release(xs->pool->fq);
 330         sock_def_readable(&xs->sk);
 331 }
 332
 333 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
 334 {
 335         u32 len = xdp_get_buff_len(xdp);
 336         int err;
 337
 338         spin_lock_bh(&xs->rx_lock);
 339         err = xsk_rcv_check(xs, xdp, len);
 340         if (!err) {
 341                 err = __xsk_rcv(xs, xdp, len);
 342                 xsk_flush(xs);
 343         }
 344         spin_unlock_bh(&xs->rx_lock);
 345         return err;
 346 }
 347
 348 static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
 349 {
 350         u32 len = xdp_get_buff_len(xdp);
 351         int err;
 352
 353         err = xsk_rcv_check(xs, xdp, len);
 354         if (err)
 355                 return err;
 356
 357         if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
 358                 len = xdp->data_end - xdp->data;
 359                 return xsk_rcv_zc(xs, xdp, len);
 360         }
 361
 362         err = __xsk_rcv(xs, xdp, len);
 363         if (!err)
 364                 xdp_return_buff(xdp);
 365         return err;
 366 }
 367
 368 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
 369 {
 370         struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
 371         int err;
 372
 373         err = xsk_rcv(xs, xdp);
 374         if (err)
 375                 return err;
 376
 377         if (!xs->flush_node.prev)
 378                 list_add(&xs->flush_node, flush_list);
 379
 380         return 0;
 381 }
 382
 383 void __xsk_map_flush(void)
 384 {
 385         struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
 386         struct xdp_sock *xs, *tmp;
 387
 388         list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
 389                 xsk_flush(xs);
 390                 __list_del_clearprev(&xs->flush_node);
 391         }
 392 }
 393
 394 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 395 {
 396         xskq_prod_submit_n(pool->cq, nb_entries);
 397 }
 398 EXPORT_SYMBOL(xsk_tx_completed);
 399
 400 void xsk_tx_release(struct xsk_buff_pool *pool)
 401 {
 402         struct xdp_sock *xs;
 403
 404         rcu_read_lock();
 405         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
 406                 __xskq_cons_release(xs->tx);
 407                 if (xsk_tx_writeable(xs))
 408                         xs->sk.sk_write_space(&xs->sk);
 409         }
 410         rcu_read_unlock();
 411 }
 412 EXPORT_SYMBOL(xsk_tx_release);
 413
 414 bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
 415 {
 416         struct xdp_sock *xs;
 417
 418         rcu_read_lock();
 419         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
 420                 if (!xskq_cons_peek_desc(xs->tx, desc, pool)) {
 421                         if (xskq_has_descs(xs->tx))
 422                                 xskq_cons_release(xs->tx);
 423                         continue;
 424                 }
 425
 426                 /* This is the backpressure mechanism for the Tx path.
 427                  * Reserve space in the completion queue and only proceed
 428                  * if there is space in it. This avoids having to implement
 429                  * any buffering in the Tx path.
 430                  */
 431                 if (xskq_prod_reserve_addr(pool->cq, desc->addr))
 432                         goto out;
 433
 434                 xskq_cons_release(xs->tx);
 435                 rcu_read_unlock();
 436                 return true;
 437         }
 438
 439 out:
 440         rcu_read_unlock();
 441         return false;
 442 }
 443 EXPORT_SYMBOL(xsk_tx_peek_desc);
 444
 445 static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries)
 446 {
 447         struct xdp_desc *descs = pool->tx_descs;
 448         u32 nb_pkts = 0;
 449
 450         while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts]))
 451                 nb_pkts++;
 452
 453         xsk_tx_release(pool);
 454         return nb_pkts;
 455 }
 456
 457 u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts)
 458 {
 459         struct xdp_sock *xs;
 460
 461         rcu_read_lock();
 462         if (!list_is_singular(&pool->xsk_tx_list)) {
 463                 /* Fallback to the non-batched version */
 464                 rcu_read_unlock();
 465                 return xsk_tx_peek_release_fallback(pool, nb_pkts);
 466         }
 467
 468         xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
 469         if (!xs) {
 470                 nb_pkts = 0;
 471                 goto out;
 472         }
 473
 474         nb_pkts = xskq_cons_nb_entries(xs->tx, nb_pkts);
 475
 476         /* This is the backpressure mechanism for the Tx path. Try to
 477          * reserve space in the completion queue for all packets, but
 478          * if there are fewer slots available, just process that many
 479          * packets. This avoids having to implement any buffering in
 480          * the Tx path.
 481          */
 482         nb_pkts = xskq_prod_nb_free(pool->cq, nb_pkts);
 483         if (!nb_pkts)
 484                 goto out;
 485
 486         nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, nb_pkts);
 487         if (!nb_pkts) {
 488                 xs->tx->queue_empty_descs++;
 489                 goto out;
 490         }
 491
 492         __xskq_cons_release(xs->tx);
 493         xskq_prod_write_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
 494         xs->sk.sk_write_space(&xs->sk);
 495
 496 out:
 497         rcu_read_unlock();
 498         return nb_pkts;
 499 }
 500 EXPORT_SYMBOL(xsk_tx_peek_release_desc_batch);
 501
 502 static int xsk_wakeup(struct xdp_sock *xs, u8 flags)
 503 {
 504         struct net_device *dev = xs->dev;
 505
 506         return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
 507 }
 508
 509 static int xsk_cq_reserve_addr_locked(struct xdp_sock *xs, u64 addr)
 510 {
 511         unsigned long flags;
 512         int ret;
 513
 514         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 515         ret = xskq_prod_reserve_addr(xs->pool->cq, addr);
 516         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 517
 518         return ret;
 519 }
 520
 521 static void xsk_cq_submit_locked(struct xdp_sock *xs, u32 n)
 522 {
 523         unsigned long flags;
 524
 525         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 526         xskq_prod_submit_n(xs->pool->cq, n);
 527         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 528 }
 529
 530 static void xsk_cq_cancel_locked(struct xdp_sock *xs, u32 n)
 531 {
 532         unsigned long flags;
 533
 534         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 535         xskq_prod_cancel_n(xs->pool->cq, n);
 536         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 537 }
 538
 539 static u32 xsk_get_num_desc(struct sk_buff *skb)
 540 {
 541         return skb ? (long)skb_shinfo(skb)->destructor_arg : 0;
 542 }
 543
 544 static void xsk_destruct_skb(struct sk_buff *skb)
 545 {
 546         xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb));
 547         sock_wfree(skb);
 548 }
 549
 550 static void xsk_set_destructor_arg(struct sk_buff *skb)
 551 {
 552         long num = xsk_get_num_desc(xdp_sk(skb->sk)->skb) + 1;
 553
 554         skb_shinfo(skb)->destructor_arg = (void *)num;
 555 }
 556
 557 static void xsk_consume_skb(struct sk_buff *skb)
 558 {
 559         struct xdp_sock *xs = xdp_sk(skb->sk);
 560
 561         skb->destructor = sock_wfree;
 562         xsk_cq_cancel_locked(xs, xsk_get_num_desc(skb));
 563         /* Free skb without triggering the perf drop trace */
 564         consume_skb(skb);
 565         xs->skb = NULL;
 566 }
 567
 568 static void xsk_drop_skb(struct sk_buff *skb)
 569 {
 570         xdp_sk(skb->sk)->tx->invalid_descs += xsk_get_num_desc(skb);
 571         xsk_consume_skb(skb);
 572 }
 573
 574 static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
 575                                               struct xdp_desc *desc)
 576 {
 577         struct xsk_buff_pool *pool = xs->pool;
 578         u32 hr, len, ts, offset, copy, copied;
 579         struct sk_buff *skb = xs->skb;
 580         struct page *page;
 581         void *buffer;
 582         int err, i;
 583         u64 addr;
 584
 585         if (!skb) {
 586                 hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom));
 587
 588                 skb = sock_alloc_send_skb(&xs->sk, hr, 1, &err);
 589                 if (unlikely(!skb))
 590                         return ERR_PTR(err);
 591
 592                 skb_reserve(skb, hr);
 593         }
 594
 595         addr = desc->addr;
 596         len = desc->len;
 597         ts = pool->unaligned ? len : pool->chunk_size;
 598
 599         buffer = xsk_buff_raw_get_data(pool, addr);
 600         offset = offset_in_page(buffer);
 601         addr = buffer - pool->addrs;
 602
 603         for (copied = 0, i = skb_shinfo(skb)->nr_frags; copied < len; i++) {
 604                 if (unlikely(i >= MAX_SKB_FRAGS))
 605                         return ERR_PTR(-EFAULT);
 606
 607                 page = pool->umem->pgs[addr >> PAGE_SHIFT];
 608                 get_page(page);
 609
 610                 copy = min_t(u32, PAGE_SIZE - offset, len - copied);
 611                 skb_fill_page_desc(skb, i, page, offset, copy);
 612
 613                 copied += copy;
 614                 addr += copy;
 615                 offset = 0;
 616         }
 617
 618         skb->len += len;
 619         skb->data_len += len;
 620         skb->truesize += ts;
 621
 622         refcount_add(ts, &xs->sk.sk_wmem_alloc);
 623
 624         return skb;
 625 }
 626
 627 static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
 628                                      struct xdp_desc *desc)
 629 {
 630         struct net_device *dev = xs->dev;
 631         struct sk_buff *skb = xs->skb;
 632         int err;
 633
 634         if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {
 635                 skb = xsk_build_skb_zerocopy(xs, desc);
 636                 if (IS_ERR(skb)) {
 637                         err = PTR_ERR(skb);
 638                         goto free_err;
 639                 }
 640         } else {
 641                 u32 hr, tr, len;
 642                 void *buffer;
 643
 644                 buffer = xsk_buff_raw_get_data(xs->pool, desc->addr);
 645                 len = desc->len;
 646
 647                 if (!skb) {
 648                         hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom));
 649                         tr = dev->needed_tailroom;
 650                         skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err);
 651                         if (unlikely(!skb))
 652                                 goto free_err;
 653
 654                         skb_reserve(skb, hr);
 655                         skb_put(skb, len);
 656
 657                         err = skb_store_bits(skb, 0, buffer, len);
 658                         if (unlikely(err))
 659                                 goto free_err;
 660                 } else {
 661                         int nr_frags = skb_shinfo(skb)->nr_frags;
 662                         struct page *page;
 663                         u8 *vaddr;
 664
 665                         if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) {
 666                                 err = -EFAULT;
 667                                 goto free_err;
 668                         }
 669
 670                         page = alloc_page(xs->sk.sk_allocation);
 671                         if (unlikely(!page)) {
 672                                 err = -EAGAIN;
 673                                 goto free_err;
 674                         }
 675
 676                         vaddr = kmap_local_page(page);
 677                         memcpy(vaddr, buffer, len);
 678                         kunmap_local(vaddr);
 679
 680                         skb_add_rx_frag(skb, nr_frags, page, 0, len, 0);
 681                 }
 682         }
 683
 684         skb->dev = dev;
 685         skb->priority = xs->sk.sk_priority;
 686         skb->mark = xs->sk.sk_mark;
 687         skb->destructor = xsk_destruct_skb;
 688         xsk_set_destructor_arg(skb);
 689
 690         return skb;
 691
 692 free_err:
 693         if (err == -EAGAIN) {
 694                 xsk_cq_cancel_locked(xs, 1);
 695         } else {
 696                 xsk_set_destructor_arg(skb);
 697                 xsk_drop_skb(skb);
 698                 xskq_cons_release(xs->tx);
 699         }
 700
 701         return ERR_PTR(err);
 702 }
 703
 704 static int __xsk_generic_xmit(struct sock *sk)
 705 {
 706         struct xdp_sock *xs = xdp_sk(sk);
 707         u32 max_batch = TX_BATCH_SIZE;
 708         bool sent_frame = false;
 709         struct xdp_desc desc;
 710         struct sk_buff *skb;
 711         int err = 0;
 712
 713         mutex_lock(&xs->mutex);
 714
 715         /* Since we dropped the RCU read lock, the socket state might have changed. */
 716         if (unlikely(!xsk_is_bound(xs))) {
 717                 err = -ENXIO;
 718                 goto out;
 719         }
 720
 721         if (xs->queue_id >= xs->dev->real_num_tx_queues)
 722                 goto out;
 723
 724         while (xskq_cons_peek_desc(xs->tx, &desc, xs->pool)) {
 725                 if (max_batch-- == 0) {
 726                         err = -EAGAIN;
 727                         goto out;
 728                 }
 729
 730                 /* This is the backpressure mechanism for the Tx path.
 731                  * Reserve space in the completion queue and only proceed
 732                  * if there is space in it. This avoids having to implement
 733                  * any buffering in the Tx path.
 734                  */
 735                 if (xsk_cq_reserve_addr_locked(xs, desc.addr))
 736                         goto out;
 737
 738                 skb = xsk_build_skb(xs, &desc);
 739                 if (IS_ERR(skb)) {
 740                         err = PTR_ERR(skb);
 741                         if (err == -EAGAIN)
 742                                 goto out;
 743                         err = 0;
 744                         continue;
 745                 }
 746
 747                 xskq_cons_release(xs->tx);
 748
 749                 if (xp_mb_desc(&desc)) {
 750                         xs->skb = skb;
 751                         continue;
 752                 }
 753
 754                 err = __dev_direct_xmit(skb, xs->queue_id);
 755                 if  (err == NETDEV_TX_BUSY) {
 756                         /* Tell user-space to retry the send */
 757                         xskq_cons_cancel_n(xs->tx, xsk_get_num_desc(skb));
 758                         xsk_consume_skb(skb);
 759                         err = -EAGAIN;
 760                         goto out;
 761                 }
 762
 763                 /* Ignore NET_XMIT_CN as packet might have been sent */
 764                 if (err == NET_XMIT_DROP) {
 765                         /* SKB completed but not sent */
 766                         err = -EBUSY;
 767                         xs->skb = NULL;
 768                         goto out;
 769                 }
 770
 771                 sent_frame = true;
 772                 xs->skb = NULL;
 773         }
 774
 775         if (xskq_has_descs(xs->tx)) {
 776                 if (xs->skb)
 777                         xsk_drop_skb(xs->skb);
 778                 xskq_cons_release(xs->tx);
 779         }
 780
 781 out:
 782         if (sent_frame)
 783                 if (xsk_tx_writeable(xs))
 784                         sk->sk_write_space(sk);
 785
 786         mutex_unlock(&xs->mutex);
 787         return err;
 788 }
 789
 790 static int xsk_generic_xmit(struct sock *sk)
 791 {
 792         int ret;
 793
 794         /* Drop the RCU lock since the SKB path might sleep. */
 795         rcu_read_unlock();
 796         ret = __xsk_generic_xmit(sk);
 797         /* Reaquire RCU lock before going into common code. */
 798         rcu_read_lock();
 799
 800         return ret;
 801 }
 802
 803 static bool xsk_no_wakeup(struct sock *sk)
 804 {
 805 #ifdef CONFIG_NET_RX_BUSY_POLL
 806         /* Prefer busy-polling, skip the wakeup. */
 807         return READ_ONCE(sk->sk_prefer_busy_poll) && READ_ONCE(sk->sk_ll_usec) &&
 808                 READ_ONCE(sk->sk_napi_id) >= MIN_NAPI_ID;
 809 #else
 810         return false;
 811 #endif
 812 }
 813
 814 static int xsk_check_common(struct xdp_sock *xs)
 815 {
 816         if (unlikely(!xsk_is_bound(xs)))
 817                 return -ENXIO;
 818         if (unlikely(!(xs->dev->flags & IFF_UP)))
 819                 return -ENETDOWN;
 820
 821         return 0;
 822 }
 823
 824 static int __xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
 825 {
 826         bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
 827         struct sock *sk = sock->sk;
 828         struct xdp_sock *xs = xdp_sk(sk);
 829         struct xsk_buff_pool *pool;
 830         int err;
 831
 832         err = xsk_check_common(xs);
 833         if (err)
 834                 return err;
 835         if (unlikely(need_wait))
 836                 return -EOPNOTSUPP;
 837         if (unlikely(!xs->tx))
 838                 return -ENOBUFS;
 839
 840         if (sk_can_busy_loop(sk)) {
 841                 if (xs->zc)
 842                         __sk_mark_napi_id_once(sk, xsk_pool_get_napi_id(xs->pool));
 843                 sk_busy_loop(sk, 1); /* only support non-blocking sockets */
 844         }
 845
 846         if (xs->zc && xsk_no_wakeup(sk))
 847                 return 0;
 848
 849         pool = xs->pool;
 850         if (pool->cached_need_wakeup & XDP_WAKEUP_TX) {
 851                 if (xs->zc)
 852                         return xsk_wakeup(xs, XDP_WAKEUP_TX);
 853                 return xsk_generic_xmit(sk);
 854         }
 855         return 0;
 856 }
 857
 858 static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
 859 {
 860         int ret;
 861
 862         rcu_read_lock();
 863         ret = __xsk_sendmsg(sock, m, total_len);
 864         rcu_read_unlock();
 865
 866         return ret;
 867 }
 868
 869 static int __xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
 870 {
 871         bool need_wait = !(flags & MSG_DONTWAIT);
 872         struct sock *sk = sock->sk;
 873         struct xdp_sock *xs = xdp_sk(sk);
 874         int err;
 875
 876         err = xsk_check_common(xs);
 877         if (err)
 878                 return err;
 879         if (unlikely(!xs->rx))
 880                 return -ENOBUFS;
 881         if (unlikely(need_wait))
 882                 return -EOPNOTSUPP;
 883
 884         if (sk_can_busy_loop(sk))
 885                 sk_busy_loop(sk, 1); /* only support non-blocking sockets */
 886
 887         if (xsk_no_wakeup(sk))
 888                 return 0;
 889
 890         if (xs->pool->cached_need_wakeup & XDP_WAKEUP_RX && xs->zc)
 891                 return xsk_wakeup(xs, XDP_WAKEUP_RX);
 892         return 0;
 893 }
 894
 895 static int xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
 896 {
 897         int ret;
 898
 899         rcu_read_lock();
 900         ret = __xsk_recvmsg(sock, m, len, flags);
 901         rcu_read_unlock();
 902
 903         return ret;
 904 }
 905
 906 static __poll_t xsk_poll(struct file *file, struct socket *sock,
 907                              struct poll_table_struct *wait)
 908 {
 909         __poll_t mask = 0;
 910         struct sock *sk = sock->sk;
 911         struct xdp_sock *xs = xdp_sk(sk);
 912         struct xsk_buff_pool *pool;
 913
 914         sock_poll_wait(file, sock, wait);
 915
 916         rcu_read_lock();
 917         if (xsk_check_common(xs))
 918                 goto skip_tx;
 919
 920         pool = xs->pool;
 921
 922         if (pool->cached_need_wakeup) {
 923                 if (xs->zc)
 924                         xsk_wakeup(xs, pool->cached_need_wakeup);
 925                 else if (xs->tx)
 926                         /* Poll needs to drive Tx also in copy mode */
 927                         xsk_generic_xmit(sk);
 928         }
 929
 930 skip_tx:
 931         if (xs->rx && !xskq_prod_is_empty(xs->rx))
 932                 mask |= EPOLLIN | EPOLLRDNORM;
 933         if (xs->tx && xsk_tx_writeable(xs))
 934                 mask |= EPOLLOUT | EPOLLWRNORM;
 935
 936         rcu_read_unlock();
 937         return mask;
 938 }
 939
 940 static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
 941                           bool umem_queue)
 942 {
 943         struct xsk_queue *q;
 944
 945         if (entries == 0 || *queue || !is_power_of_2(entries))
 946                 return -EINVAL;
 947
 948         q = xskq_create(entries, umem_queue);
 949         if (!q)
 950                 return -ENOMEM;
 951
 952         /* Make sure queue is ready before it can be seen by others */
 953         smp_wmb();
 954         WRITE_ONCE(*queue, q);
 955         return 0;
 956 }
 957
 958 static void xsk_unbind_dev(struct xdp_sock *xs)
 959 {
 960         struct net_device *dev = xs->dev;
 961
 962         if (xs->state != XSK_BOUND)
 963                 return;
 964         WRITE_ONCE(xs->state, XSK_UNBOUND);
 965
 966         /* Wait for driver to stop using the xdp socket. */
 967         xp_del_xsk(xs->pool, xs);
 968         synchronize_net();
 969         dev_put(dev);
 970 }
 971
 972 static struct xsk_map *xsk_get_map_list_entry(struct xdp_sock *xs,
 973                                               struct xdp_sock __rcu ***map_entry)
 974 {
 975         struct xsk_map *map = NULL;
 976         struct xsk_map_node *node;
 977
 978         *map_entry = NULL;
 979
 980         spin_lock_bh(&xs->map_list_lock);
 981         node = list_first_entry_or_null(&xs->map_list, struct xsk_map_node,
 982                                         node);
 983         if (node) {
 984                 bpf_map_inc(&node->map->map);
 985                 map = node->map;
 986                 *map_entry = node->map_entry;
 987         }
 988         spin_unlock_bh(&xs->map_list_lock);
 989         return map;
 990 }
 991
 992 static void xsk_delete_from_maps(struct xdp_sock *xs)
 993 {
 994         /* This function removes the current XDP socket from all the
 995          * maps it resides in. We need to take extra care here, due to
 996          * the two locks involved. Each map has a lock synchronizing
 997          * updates to the entries, and each socket has a lock that
 998          * synchronizes access to the list of maps (map_list). For
 999          * deadlock avoidance the locks need to be taken in the order
1000          * "map lock"->"socket map list lock". We start off by
1001          * accessing the socket map list, and take a reference to the
1002          * map to guarantee existence between the
1003          * xsk_get_map_list_entry() and xsk_map_try_sock_delete()
1004          * calls. Then we ask the map to remove the socket, which
1005          * tries to remove the socket from the map. Note that there
1006          * might be updates to the map between
1007          * xsk_get_map_list_entry() and xsk_map_try_sock_delete().
1008          */
1009         struct xdp_sock __rcu **map_entry = NULL;
1010         struct xsk_map *map;
1011
1012         while ((map = xsk_get_map_list_entry(xs, &map_entry))) {
1013                 xsk_map_try_sock_delete(map, xs, map_entry);
1014                 bpf_map_put(&map->map);
1015         }
1016 }
1017
1018 static int xsk_release(struct socket *sock)
1019 {
1020         struct sock *sk = sock->sk;
1021         struct xdp_sock *xs = xdp_sk(sk);
1022         struct net *net;
1023
1024         if (!sk)
1025                 return 0;
1026
1027         net = sock_net(sk);
1028
1029         if (xs->skb)
1030                 xsk_drop_skb(xs->skb);
1031
1032         mutex_lock(&net->xdp.lock);
1033         sk_del_node_init_rcu(sk);
1034         mutex_unlock(&net->xdp.lock);
1035
1036         sock_prot_inuse_add(net, sk->sk_prot, -1);
1037
1038         xsk_delete_from_maps(xs);
1039         mutex_lock(&xs->mutex);
1040         xsk_unbind_dev(xs);
1041         mutex_unlock(&xs->mutex);
1042
1043         xskq_destroy(xs->rx);
1044         xskq_destroy(xs->tx);
1045         xskq_destroy(xs->fq_tmp);
1046         xskq_destroy(xs->cq_tmp);
1047
1048         sock_orphan(sk);
1049         sock->sk = NULL;
1050
1051         sock_put(sk);
1052
1053         return 0;
1054 }
1055
1056 static struct socket *xsk_lookup_xsk_from_fd(int fd)
1057 {
1058         struct socket *sock;
1059         int err;
1060
1061         sock = sockfd_lookup(fd, &err);
1062         if (!sock)
1063                 return ERR_PTR(-ENOTSOCK);
1064
1065         if (sock->sk->sk_family != PF_XDP) {
1066                 sockfd_put(sock);
1067                 return ERR_PTR(-ENOPROTOOPT);
1068         }
1069
1070         return sock;
1071 }
1072
1073 static bool xsk_validate_queues(struct xdp_sock *xs)
1074 {
1075         return xs->fq_tmp && xs->cq_tmp;
1076 }
1077
1078 static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
1079 {
1080         struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
1081         struct sock *sk = sock->sk;
1082         struct xdp_sock *xs = xdp_sk(sk);
1083         struct net_device *dev;
1084         int bound_dev_if;
1085         u32 flags, qid;
1086         int err = 0;
1087
1088         if (addr_len < sizeof(struct sockaddr_xdp))
1089                 return -EINVAL;
1090         if (sxdp->sxdp_family != AF_XDP)
1091                 return -EINVAL;
1092
1093         flags = sxdp->sxdp_flags;
1094         if (flags & ~(XDP_SHARED_UMEM | XDP_COPY | XDP_ZEROCOPY |
1095                       XDP_USE_NEED_WAKEUP | XDP_USE_SG))
1096                 return -EINVAL;
1097
1098         bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1099         if (bound_dev_if && bound_dev_if != sxdp->sxdp_ifindex)
1100                 return -EINVAL;
1101
1102         rtnl_lock();
1103         mutex_lock(&xs->mutex);
1104         if (xs->state != XSK_READY) {
1105                 err = -EBUSY;
1106                 goto out_release;
1107         }
1108
1109         dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
1110         if (!dev) {
1111                 err = -ENODEV;
1112                 goto out_release;
1113         }
1114
1115         if (!xs->rx && !xs->tx) {
1116                 err = -EINVAL;
1117                 goto out_unlock;
1118         }
1119
1120         qid = sxdp->sxdp_queue_id;
1121
1122         if (flags & XDP_SHARED_UMEM) {
1123                 struct xdp_sock *umem_xs;
1124                 struct socket *sock;
1125
1126                 if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY) ||
1127                     (flags & XDP_USE_NEED_WAKEUP) || (flags & XDP_USE_SG)) {
1128                         /* Cannot specify flags for shared sockets. */
1129                         err = -EINVAL;
1130                         goto out_unlock;
1131                 }
1132
1133                 if (xs->umem) {
1134                         /* We have already our own. */
1135                         err = -EINVAL;
1136                         goto out_unlock;
1137                 }
1138
1139                 sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
1140                 if (IS_ERR(sock)) {
1141                         err = PTR_ERR(sock);
1142                         goto out_unlock;
1143                 }
1144
1145                 umem_xs = xdp_sk(sock->sk);
1146                 if (!xsk_is_bound(umem_xs)) {
1147                         err = -EBADF;
1148                         sockfd_put(sock);
1149                         goto out_unlock;
1150                 }
1151
1152                 if (umem_xs->queue_id != qid || umem_xs->dev != dev) {
1153                         /* Share the umem with another socket on another qid
1154                          * and/or device.
1155                          */
1156                         xs->pool = xp_create_and_assign_umem(xs,
1157                                                              umem_xs->umem);
1158                         if (!xs->pool) {
1159                                 err = -ENOMEM;
1160                                 sockfd_put(sock);
1161                                 goto out_unlock;
1162                         }
1163
1164                         err = xp_assign_dev_shared(xs->pool, umem_xs, dev,
1165                                                    qid);
1166                         if (err) {
1167                                 xp_destroy(xs->pool);
1168                                 xs->pool = NULL;
1169                                 sockfd_put(sock);
1170                                 goto out_unlock;
1171                         }
1172                 } else {
1173                         /* Share the buffer pool with the other socket. */
1174                         if (xs->fq_tmp || xs->cq_tmp) {
1175                                 /* Do not allow setting your own fq or cq. */
1176                                 err = -EINVAL;
1177                                 sockfd_put(sock);
1178                                 goto out_unlock;
1179                         }
1180
1181                         xp_get_pool(umem_xs->pool);
1182                         xs->pool = umem_xs->pool;
1183
1184                         /* If underlying shared umem was created without Tx
1185                          * ring, allocate Tx descs array that Tx batching API
1186                          * utilizes
1187                          */
1188                         if (xs->tx && !xs->pool->tx_descs) {
1189                                 err = xp_alloc_tx_descs(xs->pool, xs);
1190                                 if (err) {
1191                                         xp_put_pool(xs->pool);
1192                                         sockfd_put(sock);
1193                                         goto out_unlock;
1194                                 }
1195                         }
1196                 }
1197
1198                 xdp_get_umem(umem_xs->umem);
1199                 WRITE_ONCE(xs->umem, umem_xs->umem);
1200                 sockfd_put(sock);
1201         } else if (!xs->umem || !xsk_validate_queues(xs)) {
1202                 err = -EINVAL;
1203                 goto out_unlock;
1204         } else {
1205                 /* This xsk has its own umem. */
1206                 xs->pool = xp_create_and_assign_umem(xs, xs->umem);
1207                 if (!xs->pool) {
1208                         err = -ENOMEM;
1209                         goto out_unlock;
1210                 }
1211
1212                 err = xp_assign_dev(xs->pool, dev, qid, flags);
1213                 if (err) {
1214                         xp_destroy(xs->pool);
1215                         xs->pool = NULL;
1216                         goto out_unlock;
1217                 }
1218         }
1219
1220         /* FQ and CQ are now owned by the buffer pool and cleaned up with it. */
1221         xs->fq_tmp = NULL;
1222         xs->cq_tmp = NULL;
1223
1224         xs->dev = dev;
1225         xs->zc = xs->umem->zc;
1226         xs->sg = !!(flags & XDP_USE_SG);
1227         xs->queue_id = qid;
1228         xp_add_xsk(xs->pool, xs);
1229
1230 out_unlock:
1231         if (err) {
1232                 dev_put(dev);
1233         } else {
1234                 /* Matches smp_rmb() in bind() for shared umem
1235                  * sockets, and xsk_is_bound().
1236                  */
1237                 smp_wmb();
1238                 WRITE_ONCE(xs->state, XSK_BOUND);
1239         }
1240 out_release:
1241         mutex_unlock(&xs->mutex);
1242         rtnl_unlock();
1243         return err;
1244 }
1245
1246 struct xdp_umem_reg_v1 {
1247         __u64 addr; /* Start of packet data area */
1248         __u64 len; /* Length of packet data area */
1249         __u32 chunk_size;
1250         __u32 headroom;
1251 };
1252
1253 static int xsk_setsockopt(struct socket *sock, int level, int optname,
1254                           sockptr_t optval, unsigned int optlen)
1255 {
1256         struct sock *sk = sock->sk;
1257         struct xdp_sock *xs = xdp_sk(sk);
1258         int err;
1259
1260         if (level != SOL_XDP)
1261                 return -ENOPROTOOPT;
1262
1263         switch (optname) {
1264         case XDP_RX_RING:
1265         case XDP_TX_RING:
1266         {
1267                 struct xsk_queue **q;
1268                 int entries;
1269
1270                 if (optlen < sizeof(entries))
1271                         return -EINVAL;
1272                 if (copy_from_sockptr(&entries, optval, sizeof(entries)))
1273                         return -EFAULT;
1274
1275                 mutex_lock(&xs->mutex);
1276                 if (xs->state != XSK_READY) {
1277                         mutex_unlock(&xs->mutex);
1278                         return -EBUSY;
1279                 }
1280                 q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
1281                 err = xsk_init_queue(entries, q, false);
1282                 if (!err && optname == XDP_TX_RING)
1283                         /* Tx needs to be explicitly woken up the first time */
1284                         xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
1285                 mutex_unlock(&xs->mutex);
1286                 return err;
1287         }
1288         case XDP_UMEM_REG:
1289         {
1290                 size_t mr_size = sizeof(struct xdp_umem_reg);
1291                 struct xdp_umem_reg mr = {};
1292                 struct xdp_umem *umem;
1293
1294                 if (optlen < sizeof(struct xdp_umem_reg_v1))
1295                         return -EINVAL;
1296                 else if (optlen < sizeof(mr))
1297                         mr_size = sizeof(struct xdp_umem_reg_v1);
1298
1299                 if (copy_from_sockptr(&mr, optval, mr_size))
1300                         return -EFAULT;
1301
1302                 mutex_lock(&xs->mutex);
1303                 if (xs->state != XSK_READY || xs->umem) {
1304                         mutex_unlock(&xs->mutex);
1305                         return -EBUSY;
1306                 }
1307
1308                 umem = xdp_umem_create(&mr);
1309                 if (IS_ERR(umem)) {
1310                         mutex_unlock(&xs->mutex);
1311                         return PTR_ERR(umem);
1312                 }
1313
1314                 /* Make sure umem is ready before it can be seen by others */
1315                 smp_wmb();
1316                 WRITE_ONCE(xs->umem, umem);
1317                 mutex_unlock(&xs->mutex);
1318                 return 0;
1319         }
1320         case XDP_UMEM_FILL_RING:
1321         case XDP_UMEM_COMPLETION_RING:
1322         {
1323                 struct xsk_queue **q;
1324                 int entries;
1325
1326                 if (copy_from_sockptr(&entries, optval, sizeof(entries)))
1327                         return -EFAULT;
1328
1329                 mutex_lock(&xs->mutex);
1330                 if (xs->state != XSK_READY) {
1331                         mutex_unlock(&xs->mutex);
1332                         return -EBUSY;
1333                 }
1334
1335                 q = (optname == XDP_UMEM_FILL_RING) ? &xs->fq_tmp :
1336                         &xs->cq_tmp;
1337                 err = xsk_init_queue(entries, q, true);
1338                 mutex_unlock(&xs->mutex);
1339                 return err;
1340         }
1341         default:
1342                 break;
1343         }
1344
1345         return -ENOPROTOOPT;
1346 }
1347
1348 static void xsk_enter_rxtx_offsets(struct xdp_ring_offset_v1 *ring)
1349 {
1350         ring->producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
1351         ring->consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
1352         ring->desc = offsetof(struct xdp_rxtx_ring, desc);
1353 }
1354
1355 static void xsk_enter_umem_offsets(struct xdp_ring_offset_v1 *ring)
1356 {
1357         ring->producer = offsetof(struct xdp_umem_ring, ptrs.producer);
1358         ring->consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
1359         ring->desc = offsetof(struct xdp_umem_ring, desc);
1360 }
1361
1362 struct xdp_statistics_v1 {
1363         __u64 rx_dropped;
1364         __u64 rx_invalid_descs;
1365         __u64 tx_invalid_descs;
1366 };
1367
1368 static int xsk_getsockopt(struct socket *sock, int level, int optname,
1369                           char __user *optval, int __user *optlen)
1370 {
1371         struct sock *sk = sock->sk;
1372         struct xdp_sock *xs = xdp_sk(sk);
1373         int len;
1374
1375         if (level != SOL_XDP)
1376                 return -ENOPROTOOPT;
1377
1378         if (get_user(len, optlen))
1379                 return -EFAULT;
1380         if (len < 0)
1381                 return -EINVAL;
1382
1383         switch (optname) {
1384         case XDP_STATISTICS:
1385         {
1386                 struct xdp_statistics stats = {};
1387                 bool extra_stats = true;
1388                 size_t stats_size;
1389
1390                 if (len < sizeof(struct xdp_statistics_v1)) {
1391                         return -EINVAL;
1392                 } else if (len < sizeof(stats)) {
1393                         extra_stats = false;
1394                         stats_size = sizeof(struct xdp_statistics_v1);
1395                 } else {
1396                         stats_size = sizeof(stats);
1397                 }
1398
1399                 mutex_lock(&xs->mutex);
1400                 stats.rx_dropped = xs->rx_dropped;
1401                 if (extra_stats) {
1402                         stats.rx_ring_full = xs->rx_queue_full;
1403                         stats.rx_fill_ring_empty_descs =
1404                                 xs->pool ? xskq_nb_queue_empty_descs(xs->pool->fq) : 0;
1405                         stats.tx_ring_empty_descs = xskq_nb_queue_empty_descs(xs->tx);
1406                 } else {
1407                         stats.rx_dropped += xs->rx_queue_full;
1408                 }
1409                 stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
1410                 stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
1411                 mutex_unlock(&xs->mutex);
1412
1413                 if (copy_to_user(optval, &stats, stats_size))
1414                         return -EFAULT;
1415                 if (put_user(stats_size, optlen))
1416                         return -EFAULT;
1417
1418                 return 0;
1419         }
1420         case XDP_MMAP_OFFSETS:
1421         {
1422                 struct xdp_mmap_offsets off;
1423                 struct xdp_mmap_offsets_v1 off_v1;
1424                 bool flags_supported = true;
1425                 void *to_copy;
1426
1427                 if (len < sizeof(off_v1))
1428                         return -EINVAL;
1429                 else if (len < sizeof(off))
1430                         flags_supported = false;
1431
1432                 if (flags_supported) {
1433                         /* xdp_ring_offset is identical to xdp_ring_offset_v1
1434                          * except for the flags field added to the end.
1435                          */
1436                         xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
1437                                                &off.rx);
1438                         xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
1439                                                &off.tx);
1440                         xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
1441                                                &off.fr);
1442                         xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
1443                                                &off.cr);
1444                         off.rx.flags = offsetof(struct xdp_rxtx_ring,
1445                                                 ptrs.flags);
1446                         off.tx.flags = offsetof(struct xdp_rxtx_ring,
1447                                                 ptrs.flags);
1448                         off.fr.flags = offsetof(struct xdp_umem_ring,
1449                                                 ptrs.flags);
1450                         off.cr.flags = offsetof(struct xdp_umem_ring,
1451                                                 ptrs.flags);
1452
1453                         len = sizeof(off);
1454                         to_copy = &off;
1455                 } else {
1456                         xsk_enter_rxtx_offsets(&off_v1.rx);
1457                         xsk_enter_rxtx_offsets(&off_v1.tx);
1458                         xsk_enter_umem_offsets(&off_v1.fr);
1459                         xsk_enter_umem_offsets(&off_v1.cr);
1460
1461                         len = sizeof(off_v1);
1462                         to_copy = &off_v1;
1463                 }
1464
1465                 if (copy_to_user(optval, to_copy, len))
1466                         return -EFAULT;
1467                 if (put_user(len, optlen))
1468                         return -EFAULT;
1469
1470                 return 0;
1471         }
1472         case XDP_OPTIONS:
1473         {
1474                 struct xdp_options opts = {};
1475
1476                 if (len < sizeof(opts))
1477                         return -EINVAL;
1478
1479                 mutex_lock(&xs->mutex);
1480                 if (xs->zc)
1481                         opts.flags |= XDP_OPTIONS_ZEROCOPY;
1482                 mutex_unlock(&xs->mutex);
1483
1484                 len = sizeof(opts);
1485                 if (copy_to_user(optval, &opts, len))
1486                         return -EFAULT;
1487                 if (put_user(len, optlen))
1488                         return -EFAULT;
1489
1490                 return 0;
1491         }
1492         default:
1493                 break;
1494         }
1495
1496         return -EOPNOTSUPP;
1497 }
1498
1499 static int xsk_mmap(struct file *file, struct socket *sock,
1500                     struct vm_area_struct *vma)
1501 {
1502         loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1503         unsigned long size = vma->vm_end - vma->vm_start;
1504         struct xdp_sock *xs = xdp_sk(sock->sk);
1505         int state = READ_ONCE(xs->state);
1506         struct xsk_queue *q = NULL;
1507
1508         if (state != XSK_READY && state != XSK_BOUND)
1509                 return -EBUSY;
1510
1511         if (offset == XDP_PGOFF_RX_RING) {
1512                 q = READ_ONCE(xs->rx);
1513         } else if (offset == XDP_PGOFF_TX_RING) {
1514                 q = READ_ONCE(xs->tx);
1515         } else {
1516                 /* Matches the smp_wmb() in XDP_UMEM_REG */
1517                 smp_rmb();
1518                 if (offset == XDP_UMEM_PGOFF_FILL_RING)
1519                         q = state == XSK_READY ? READ_ONCE(xs->fq_tmp) :
1520                                                  READ_ONCE(xs->pool->fq);
1521                 else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
1522                         q = state == XSK_READY ? READ_ONCE(xs->cq_tmp) :
1523                                                  READ_ONCE(xs->pool->cq);
1524         }
1525
1526         if (!q)
1527                 return -EINVAL;
1528
1529         /* Matches the smp_wmb() in xsk_init_queue */
1530         smp_rmb();
1531         if (size > q->ring_vmalloc_size)
1532                 return -EINVAL;
1533
1534         return remap_vmalloc_range(vma, q->ring, 0);
1535 }
1536
1537 static int xsk_notifier(struct notifier_block *this,
1538                         unsigned long msg, void *ptr)
1539 {
1540         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1541         struct net *net = dev_net(dev);
1542         struct sock *sk;
1543
1544         switch (msg) {
1545         case NETDEV_UNREGISTER:
1546                 mutex_lock(&net->xdp.lock);
1547                 sk_for_each(sk, &net->xdp.list) {
1548                         struct xdp_sock *xs = xdp_sk(sk);
1549
1550                         mutex_lock(&xs->mutex);
1551                         if (xs->dev == dev) {
1552                                 sk->sk_err = ENETDOWN;
1553                                 if (!sock_flag(sk, SOCK_DEAD))
1554                                         sk_error_report(sk);
1555
1556                                 xsk_unbind_dev(xs);
1557
1558                                 /* Clear device references. */
1559                                 xp_clear_dev(xs->pool);
1560                         }
1561                         mutex_unlock(&xs->mutex);
1562                 }
1563                 mutex_unlock(&net->xdp.lock);
1564                 break;
1565         }
1566         return NOTIFY_DONE;
1567 }
1568
1569 static struct proto xsk_proto = {
1570         .name =         "XDP",
1571         .owner =        THIS_MODULE,
1572         .obj_size =     sizeof(struct xdp_sock),
1573 };
1574
1575 static const struct proto_ops xsk_proto_ops = {
1576         .family         = PF_XDP,
1577         .owner          = THIS_MODULE,
1578         .release        = xsk_release,
1579         .bind           = xsk_bind,
1580         .connect        = sock_no_connect,
1581         .socketpair     = sock_no_socketpair,
1582         .accept         = sock_no_accept,
1583         .getname        = sock_no_getname,
1584         .poll           = xsk_poll,
1585         .ioctl          = sock_no_ioctl,
1586         .listen         = sock_no_listen,
1587         .shutdown       = sock_no_shutdown,
1588         .setsockopt     = xsk_setsockopt,
1589         .getsockopt     = xsk_getsockopt,
1590         .sendmsg        = xsk_sendmsg,
1591         .recvmsg        = xsk_recvmsg,
1592         .mmap           = xsk_mmap,
1593 };
1594
1595 static void xsk_destruct(struct sock *sk)
1596 {
1597         struct xdp_sock *xs = xdp_sk(sk);
1598
1599         if (!sock_flag(sk, SOCK_DEAD))
1600                 return;
1601
1602         if (!xp_put_pool(xs->pool))
1603                 xdp_put_umem(xs->umem, !xs->pool);
1604 }
1605
1606 static int xsk_create(struct net *net, struct socket *sock, int protocol,
1607                       int kern)
1608 {
1609         struct xdp_sock *xs;
1610         struct sock *sk;
1611
1612         if (!ns_capable(net->user_ns, CAP_NET_RAW))
1613                 return -EPERM;
1614         if (sock->type != SOCK_RAW)
1615                 return -ESOCKTNOSUPPORT;
1616
1617         if (protocol)
1618                 return -EPROTONOSUPPORT;
1619
1620         sock->state = SS_UNCONNECTED;
1621
1622         sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
1623         if (!sk)
1624                 return -ENOBUFS;
1625
1626         sock->ops = &xsk_proto_ops;
1627
1628         sock_init_data(sock, sk);
1629
1630         sk->sk_family = PF_XDP;
1631
1632         sk->sk_destruct = xsk_destruct;
1633
1634         sock_set_flag(sk, SOCK_RCU_FREE);
1635
1636         xs = xdp_sk(sk);
1637         xs->state = XSK_READY;
1638         mutex_init(&xs->mutex);
1639         spin_lock_init(&xs->rx_lock);
1640
1641         INIT_LIST_HEAD(&xs->map_list);
1642         spin_lock_init(&xs->map_list_lock);
1643
1644         mutex_lock(&net->xdp.lock);
1645         sk_add_node_rcu(sk, &net->xdp.list);
1646         mutex_unlock(&net->xdp.lock);
1647
1648         sock_prot_inuse_add(net, &xsk_proto, 1);
1649
1650         return 0;
1651 }
1652
1653 static const struct net_proto_family xsk_family_ops = {
1654         .family = PF_XDP,
1655         .create = xsk_create,
1656         .owner  = THIS_MODULE,
1657 };
1658
1659 static struct notifier_block xsk_netdev_notifier = {
1660         .notifier_call  = xsk_notifier,
1661 };
1662
1663 static int __net_init xsk_net_init(struct net *net)
1664 {
1665         mutex_init(&net->xdp.lock);
1666         INIT_HLIST_HEAD(&net->xdp.list);
1667         return 0;
1668 }
1669
1670 static void __net_exit xsk_net_exit(struct net *net)
1671 {
1672         WARN_ON_ONCE(!hlist_empty(&net->xdp.list));
1673 }
1674
1675 static struct pernet_operations xsk_net_ops = {
1676         .init = xsk_net_init,
1677         .exit = xsk_net_exit,
1678 };
1679
1680 static int __init xsk_init(void)
1681 {
1682         int err, cpu;
1683
1684         err = proto_register(&xsk_proto, 0 /* no slab */);
1685         if (err)
1686                 goto out;
1687
1688         err = sock_register(&xsk_family_ops);
1689         if (err)
1690                 goto out_proto;
1691
1692         err = register_pernet_subsys(&xsk_net_ops);
1693         if (err)
1694                 goto out_sk;
1695
1696         err = register_netdevice_notifier(&xsk_netdev_notifier);
1697         if (err)
1698                 goto out_pernet;
1699
1700         for_each_possible_cpu(cpu)
1701                 INIT_LIST_HEAD(&per_cpu(xskmap_flush_list, cpu));
1702         return 0;
1703
1704 out_pernet:
1705         unregister_pernet_subsys(&xsk_net_ops);
1706 out_sk:
1707         sock_unregister(PF_XDP);
1708 out_proto:
1709         proto_unregister(&xsk_proto);
1710 out:
1711         return err;
1712 }
1713
1714 fs_initcall(xsk_init);