net/xdp/xsk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* XDP sockets
   3  *
   4  * AF_XDP sockets allows a channel between XDP programs and userspace
   5  * applications.
   6  * Copyright(c) 2018 Intel Corporation.
   7  *
   8  * Author(s): Björn Töpel <bjorn.topel@intel.com>
   9  *            Magnus Karlsson <magnus.karlsson@intel.com>
  10  */
  11
  12 #define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
  13
  14 #include <linux/if_xdp.h>
  15 #include <linux/init.h>
  16 #include <linux/sched/mm.h>
  17 #include <linux/sched/signal.h>
  18 #include <linux/sched/task.h>
  19 #include <linux/socket.h>
  20 #include <linux/file.h>
  21 #include <linux/uaccess.h>
  22 #include <linux/net.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/rculist.h>
  25 #include <linux/vmalloc.h>
  26 #include <net/xdp_sock_drv.h>
  27 #include <net/busy_poll.h>
  28 #include <net/netdev_rx_queue.h>
  29 #include <net/xdp.h>
  30
  31 #include "xsk_queue.h"
  32 #include "xdp_umem.h"
  33 #include "xsk.h"
  34
  35 #define TX_BATCH_SIZE 32
  36
  37 static DEFINE_PER_CPU(struct list_head, xskmap_flush_list);
  38
  39 void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
  40 {
  41         if (pool->cached_need_wakeup & XDP_WAKEUP_RX)
  42                 return;
  43
  44         pool->fq->ring->flags |= XDP_RING_NEED_WAKEUP;
  45         pool->cached_need_wakeup |= XDP_WAKEUP_RX;
  46 }
  47 EXPORT_SYMBOL(xsk_set_rx_need_wakeup);
  48
  49 void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
  50 {
  51         struct xdp_sock *xs;
  52
  53         if (pool->cached_need_wakeup & XDP_WAKEUP_TX)
  54                 return;
  55
  56         rcu_read_lock();
  57         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
  58                 xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
  59         }
  60         rcu_read_unlock();
  61
  62         pool->cached_need_wakeup |= XDP_WAKEUP_TX;
  63 }
  64 EXPORT_SYMBOL(xsk_set_tx_need_wakeup);
  65
  66 void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
  67 {
  68         if (!(pool->cached_need_wakeup & XDP_WAKEUP_RX))
  69                 return;
  70
  71         pool->fq->ring->flags &= ~XDP_RING_NEED_WAKEUP;
  72         pool->cached_need_wakeup &= ~XDP_WAKEUP_RX;
  73 }
  74 EXPORT_SYMBOL(xsk_clear_rx_need_wakeup);
  75
  76 void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
  77 {
  78         struct xdp_sock *xs;
  79
  80         if (!(pool->cached_need_wakeup & XDP_WAKEUP_TX))
  81                 return;
  82
  83         rcu_read_lock();
  84         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
  85                 xs->tx->ring->flags &= ~XDP_RING_NEED_WAKEUP;
  86         }
  87         rcu_read_unlock();
  88
  89         pool->cached_need_wakeup &= ~XDP_WAKEUP_TX;
  90 }
  91 EXPORT_SYMBOL(xsk_clear_tx_need_wakeup);
  92
  93 bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
  94 {
  95         return pool->uses_need_wakeup;
  96 }
  97 EXPORT_SYMBOL(xsk_uses_need_wakeup);
  98
  99 struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
 100                                             u16 queue_id)
 101 {
 102         if (queue_id < dev->real_num_rx_queues)
 103                 return dev->_rx[queue_id].pool;
 104         if (queue_id < dev->real_num_tx_queues)
 105                 return dev->_tx[queue_id].pool;
 106
 107         return NULL;
 108 }
 109 EXPORT_SYMBOL(xsk_get_pool_from_qid);
 110
 111 void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id)
 112 {
 113         if (queue_id < dev->num_rx_queues)
 114                 dev->_rx[queue_id].pool = NULL;
 115         if (queue_id < dev->num_tx_queues)
 116                 dev->_tx[queue_id].pool = NULL;
 117 }
 118
 119 /* The buffer pool is stored both in the _rx struct and the _tx struct as we do
 120  * not know if the device has more tx queues than rx, or the opposite.
 121  * This might also change during run time.
 122  */
 123 int xsk_reg_pool_at_qid(struct net_device *dev, struct xsk_buff_pool *pool,
 124                         u16 queue_id)
 125 {
 126         if (queue_id >= max_t(unsigned int,
 127                               dev->real_num_rx_queues,
 128                               dev->real_num_tx_queues))
 129                 return -EINVAL;
 130
 131         if (queue_id < dev->real_num_rx_queues)
 132                 dev->_rx[queue_id].pool = pool;
 133         if (queue_id < dev->real_num_tx_queues)
 134                 dev->_tx[queue_id].pool = pool;
 135
 136         return 0;
 137 }
 138
 139 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len,
 140                         u32 flags)
 141 {
 142         u64 addr;
 143         int err;
 144
 145         addr = xp_get_handle(xskb);
 146         err = xskq_prod_reserve_desc(xs->rx, addr, len, flags);
 147         if (err) {
 148                 xs->rx_queue_full++;
 149                 return err;
 150         }
 151
 152         xp_release(xskb);
 153         return 0;
 154 }
 155
 156 static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 157 {
 158         struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
 159         u32 frags = xdp_buff_has_frags(xdp);
 160         struct xdp_buff_xsk *pos, *tmp;
 161         struct list_head *xskb_list;
 162         u32 contd = 0;
 163         int err;
 164
 165         if (frags)
 166                 contd = XDP_PKT_CONTD;
 167
 168         err = __xsk_rcv_zc(xs, xskb, len, contd);
 169         if (err)
 170                 goto err;
 171         if (likely(!frags))
 172                 return 0;
 173
 174         xskb_list = &xskb->pool->xskb_list;
 175         list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) {
 176                 if (list_is_singular(xskb_list))
 177                         contd = 0;
 178                 len = pos->xdp.data_end - pos->xdp.data;
 179                 err = __xsk_rcv_zc(xs, pos, len, contd);
 180                 if (err)
 181                         goto err;
 182                 list_del(&pos->xskb_list_node);
 183         }
 184
 185         return 0;
 186 err:
 187         xsk_buff_free(xdp);
 188         return err;
 189 }
 190
 191 static void *xsk_copy_xdp_start(struct xdp_buff *from)
 192 {
 193         if (unlikely(xdp_data_meta_unsupported(from)))
 194                 return from->data;
 195         else
 196                 return from->data_meta;
 197 }
 198
 199 static u32 xsk_copy_xdp(void *to, void **from, u32 to_len,
 200                         u32 *from_len, skb_frag_t **frag, u32 rem)
 201 {
 202         u32 copied = 0;
 203
 204         while (1) {
 205                 u32 copy_len = min_t(u32, *from_len, to_len);
 206
 207                 memcpy(to, *from, copy_len);
 208                 copied += copy_len;
 209                 if (rem == copied)
 210                         return copied;
 211
 212                 if (*from_len == copy_len) {
 213                         *from = skb_frag_address(*frag);
 214                         *from_len = skb_frag_size((*frag)++);
 215                 } else {
 216                         *from += copy_len;
 217                         *from_len -= copy_len;
 218                 }
 219                 if (to_len == copy_len)
 220                         return copied;
 221
 222                 to_len -= copy_len;
 223                 to += copy_len;
 224         }
 225 }
 226
 227 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 228 {
 229         u32 frame_size = xsk_pool_get_rx_frame_size(xs->pool);
 230         void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
 231         u32 from_len, meta_len, rem, num_desc;
 232         struct xdp_buff_xsk *xskb;
 233         struct xdp_buff *xsk_xdp;
 234         skb_frag_t *frag;
 235
 236         from_len = xdp->data_end - copy_from;
 237         meta_len = xdp->data - copy_from;
 238         rem = len + meta_len;
 239
 240         if (len <= frame_size && !xdp_buff_has_frags(xdp)) {
 241                 int err;
 242
 243                 xsk_xdp = xsk_buff_alloc(xs->pool);
 244                 if (!xsk_xdp) {
 245                         xs->rx_dropped++;
 246                         return -ENOMEM;
 247                 }
 248                 memcpy(xsk_xdp->data - meta_len, copy_from, rem);
 249                 xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 250                 err = __xsk_rcv_zc(xs, xskb, len, 0);
 251                 if (err) {
 252                         xsk_buff_free(xsk_xdp);
 253                         return err;
 254                 }
 255
 256                 return 0;
 257         }
 258
 259         num_desc = (len - 1) / frame_size + 1;
 260
 261         if (!xsk_buff_can_alloc(xs->pool, num_desc)) {
 262                 xs->rx_dropped++;
 263                 return -ENOMEM;
 264         }
 265         if (xskq_prod_nb_free(xs->rx, num_desc) < num_desc) {
 266                 xs->rx_queue_full++;
 267                 return -ENOBUFS;
 268         }
 269
 270         if (xdp_buff_has_frags(xdp)) {
 271                 struct skb_shared_info *sinfo;
 272
 273                 sinfo = xdp_get_shared_info_from_buff(xdp);
 274                 frag =  &sinfo->frags[0];
 275         }
 276
 277         do {
 278                 u32 to_len = frame_size + meta_len;
 279                 u32 copied;
 280
 281                 xsk_xdp = xsk_buff_alloc(xs->pool);
 282                 copy_to = xsk_xdp->data - meta_len;
 283
 284                 copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
 285                 rem -= copied;
 286
 287                 xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 288                 __xsk_rcv_zc(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0);
 289                 meta_len = 0;
 290         } while (rem);
 291
 292         return 0;
 293 }
 294
 295 static bool xsk_tx_writeable(struct xdp_sock *xs)
 296 {
 297         if (xskq_cons_present_entries(xs->tx) > xs->tx->nentries / 2)
 298                 return false;
 299
 300         return true;
 301 }
 302
 303 static bool xsk_is_bound(struct xdp_sock *xs)
 304 {
 305         if (READ_ONCE(xs->state) == XSK_BOUND) {
 306                 /* Matches smp_wmb() in bind(). */
 307                 smp_rmb();
 308                 return true;
 309         }
 310         return false;
 311 }
 312
 313 static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 314 {
 315         if (!xsk_is_bound(xs))
 316                 return -ENXIO;
 317
 318         if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
 319                 return -EINVAL;
 320
 321         if (len > xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) {
 322                 xs->rx_dropped++;
 323                 return -ENOSPC;
 324         }
 325
 326         sk_mark_napi_id_once_xdp(&xs->sk, xdp);
 327         return 0;
 328 }
 329
 330 static void xsk_flush(struct xdp_sock *xs)
 331 {
 332         xskq_prod_submit(xs->rx);
 333         __xskq_cons_release(xs->pool->fq);
 334         sock_def_readable(&xs->sk);
 335 }
 336
 337 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
 338 {
 339         u32 len = xdp_get_buff_len(xdp);
 340         int err;
 341
 342         spin_lock_bh(&xs->rx_lock);
 343         err = xsk_rcv_check(xs, xdp, len);
 344         if (!err) {
 345                 err = __xsk_rcv(xs, xdp, len);
 346                 xsk_flush(xs);
 347         }
 348         spin_unlock_bh(&xs->rx_lock);
 349         return err;
 350 }
 351
 352 static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
 353 {
 354         u32 len = xdp_get_buff_len(xdp);
 355         int err;
 356
 357         err = xsk_rcv_check(xs, xdp, len);
 358         if (err)
 359                 return err;
 360
 361         if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
 362                 len = xdp->data_end - xdp->data;
 363                 return xsk_rcv_zc(xs, xdp, len);
 364         }
 365
 366         err = __xsk_rcv(xs, xdp, len);
 367         if (!err)
 368                 xdp_return_buff(xdp);
 369         return err;
 370 }
 371
 372 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
 373 {
 374         struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
 375         int err;
 376
 377         err = xsk_rcv(xs, xdp);
 378         if (err)
 379                 return err;
 380
 381         if (!xs->flush_node.prev)
 382                 list_add(&xs->flush_node, flush_list);
 383
 384         return 0;
 385 }
 386
 387 void __xsk_map_flush(void)
 388 {
 389         struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
 390         struct xdp_sock *xs, *tmp;
 391
 392         list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
 393                 xsk_flush(xs);
 394                 __list_del_clearprev(&xs->flush_node);
 395         }
 396 }
 397
 398 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 399 {
 400         xskq_prod_submit_n(pool->cq, nb_entries);
 401 }
 402 EXPORT_SYMBOL(xsk_tx_completed);
 403
 404 void xsk_tx_release(struct xsk_buff_pool *pool)
 405 {
 406         struct xdp_sock *xs;
 407
 408         rcu_read_lock();
 409         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
 410                 __xskq_cons_release(xs->tx);
 411                 if (xsk_tx_writeable(xs))
 412                         xs->sk.sk_write_space(&xs->sk);
 413         }
 414         rcu_read_unlock();
 415 }
 416 EXPORT_SYMBOL(xsk_tx_release);
 417
 418 bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
 419 {
 420         struct xdp_sock *xs;
 421
 422         rcu_read_lock();
 423         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
 424                 if (!xskq_cons_peek_desc(xs->tx, desc, pool)) {
 425                         if (xskq_has_descs(xs->tx))
 426                                 xskq_cons_release(xs->tx);
 427                         continue;
 428                 }
 429
 430                 /* This is the backpressure mechanism for the Tx path.
 431                  * Reserve space in the completion queue and only proceed
 432                  * if there is space in it. This avoids having to implement
 433                  * any buffering in the Tx path.
 434                  */
 435                 if (xskq_prod_reserve_addr(pool->cq, desc->addr))
 436                         goto out;
 437
 438                 xskq_cons_release(xs->tx);
 439                 rcu_read_unlock();
 440                 return true;
 441         }
 442
 443 out:
 444         rcu_read_unlock();
 445         return false;
 446 }
 447 EXPORT_SYMBOL(xsk_tx_peek_desc);
 448
 449 static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries)
 450 {
 451         struct xdp_desc *descs = pool->tx_descs;
 452         u32 nb_pkts = 0;
 453
 454         while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts]))
 455                 nb_pkts++;
 456
 457         xsk_tx_release(pool);
 458         return nb_pkts;
 459 }
 460
 461 u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts)
 462 {
 463         struct xdp_sock *xs;
 464
 465         rcu_read_lock();
 466         if (!list_is_singular(&pool->xsk_tx_list)) {
 467                 /* Fallback to the non-batched version */
 468                 rcu_read_unlock();
 469                 return xsk_tx_peek_release_fallback(pool, nb_pkts);
 470         }
 471
 472         xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
 473         if (!xs) {
 474                 nb_pkts = 0;
 475                 goto out;
 476         }
 477
 478         nb_pkts = xskq_cons_nb_entries(xs->tx, nb_pkts);
 479
 480         /* This is the backpressure mechanism for the Tx path. Try to
 481          * reserve space in the completion queue for all packets, but
 482          * if there are fewer slots available, just process that many
 483          * packets. This avoids having to implement any buffering in
 484          * the Tx path.
 485          */
 486         nb_pkts = xskq_prod_nb_free(pool->cq, nb_pkts);
 487         if (!nb_pkts)
 488                 goto out;
 489
 490         nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, nb_pkts);
 491         if (!nb_pkts) {
 492                 xs->tx->queue_empty_descs++;
 493                 goto out;
 494         }
 495
 496         __xskq_cons_release(xs->tx);
 497         xskq_prod_write_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
 498         xs->sk.sk_write_space(&xs->sk);
 499
 500 out:
 501         rcu_read_unlock();
 502         return nb_pkts;
 503 }
 504 EXPORT_SYMBOL(xsk_tx_peek_release_desc_batch);
 505
 506 static int xsk_wakeup(struct xdp_sock *xs, u8 flags)
 507 {
 508         struct net_device *dev = xs->dev;
 509
 510         return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
 511 }
 512
 513 static int xsk_cq_reserve_addr_locked(struct xdp_sock *xs, u64 addr)
 514 {
 515         unsigned long flags;
 516         int ret;
 517
 518         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 519         ret = xskq_prod_reserve_addr(xs->pool->cq, addr);
 520         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 521
 522         return ret;
 523 }
 524
 525 static void xsk_cq_submit_locked(struct xdp_sock *xs, u32 n)
 526 {
 527         unsigned long flags;
 528
 529         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 530         xskq_prod_submit_n(xs->pool->cq, n);
 531         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 532 }
 533
 534 static void xsk_cq_cancel_locked(struct xdp_sock *xs, u32 n)
 535 {
 536         unsigned long flags;
 537
 538         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 539         xskq_prod_cancel_n(xs->pool->cq, n);
 540         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 541 }
 542
 543 static u32 xsk_get_num_desc(struct sk_buff *skb)
 544 {
 545         return skb ? (long)skb_shinfo(skb)->destructor_arg : 0;
 546 }
 547
 548 static void xsk_destruct_skb(struct sk_buff *skb)
 549 {
 550         xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb));
 551         sock_wfree(skb);
 552 }
 553
 554 static void xsk_set_destructor_arg(struct sk_buff *skb)
 555 {
 556         long num = xsk_get_num_desc(xdp_sk(skb->sk)->skb) + 1;
 557
 558         skb_shinfo(skb)->destructor_arg = (void *)num;
 559 }
 560
 561 static void xsk_consume_skb(struct sk_buff *skb)
 562 {
 563         struct xdp_sock *xs = xdp_sk(skb->sk);
 564
 565         skb->destructor = sock_wfree;
 566         xsk_cq_cancel_locked(xs, xsk_get_num_desc(skb));
 567         /* Free skb without triggering the perf drop trace */
 568         consume_skb(skb);
 569         xs->skb = NULL;
 570 }
 571
 572 static void xsk_drop_skb(struct sk_buff *skb)
 573 {
 574         xdp_sk(skb->sk)->tx->invalid_descs += xsk_get_num_desc(skb);
 575         xsk_consume_skb(skb);
 576 }
 577
 578 static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
 579                                               struct xdp_desc *desc)
 580 {
 581         struct xsk_buff_pool *pool = xs->pool;
 582         u32 hr, len, ts, offset, copy, copied;
 583         struct sk_buff *skb = xs->skb;
 584         struct page *page;
 585         void *buffer;
 586         int err, i;
 587         u64 addr;
 588
 589         if (!skb) {
 590                 hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom));
 591
 592                 skb = sock_alloc_send_skb(&xs->sk, hr, 1, &err);
 593                 if (unlikely(!skb))
 594                         return ERR_PTR(err);
 595
 596                 skb_reserve(skb, hr);
 597         }
 598
 599         addr = desc->addr;
 600         len = desc->len;
 601         ts = pool->unaligned ? len : pool->chunk_size;
 602
 603         buffer = xsk_buff_raw_get_data(pool, addr);
 604         offset = offset_in_page(buffer);
 605         addr = buffer - pool->addrs;
 606
 607         for (copied = 0, i = skb_shinfo(skb)->nr_frags; copied < len; i++) {
 608                 if (unlikely(i >= MAX_SKB_FRAGS))
 609                         return ERR_PTR(-EOVERFLOW);
 610
 611                 page = pool->umem->pgs[addr >> PAGE_SHIFT];
 612                 get_page(page);
 613
 614                 copy = min_t(u32, PAGE_SIZE - offset, len - copied);
 615                 skb_fill_page_desc(skb, i, page, offset, copy);
 616
 617                 copied += copy;
 618                 addr += copy;
 619                 offset = 0;
 620         }
 621
 622         skb->len += len;
 623         skb->data_len += len;
 624         skb->truesize += ts;
 625
 626         refcount_add(ts, &xs->sk.sk_wmem_alloc);
 627
 628         return skb;
 629 }
 630
 631 static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
 632                                      struct xdp_desc *desc)
 633 {
 634         struct net_device *dev = xs->dev;
 635         struct sk_buff *skb = xs->skb;
 636         int err;
 637
 638         if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {
 639                 skb = xsk_build_skb_zerocopy(xs, desc);
 640                 if (IS_ERR(skb)) {
 641                         err = PTR_ERR(skb);
 642                         goto free_err;
 643                 }
 644         } else {
 645                 u32 hr, tr, len;
 646                 void *buffer;
 647
 648                 buffer = xsk_buff_raw_get_data(xs->pool, desc->addr);
 649                 len = desc->len;
 650
 651                 if (!skb) {
 652                         hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom));
 653                         tr = dev->needed_tailroom;
 654                         skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err);
 655                         if (unlikely(!skb))
 656                                 goto free_err;
 657
 658                         skb_reserve(skb, hr);
 659                         skb_put(skb, len);
 660
 661                         err = skb_store_bits(skb, 0, buffer, len);
 662                         if (unlikely(err)) {
 663                                 kfree_skb(skb);
 664                                 goto free_err;
 665                         }
 666                 } else {
 667                         int nr_frags = skb_shinfo(skb)->nr_frags;
 668                         struct page *page;
 669                         u8 *vaddr;
 670
 671                         if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) {
 672                                 err = -EOVERFLOW;
 673                                 goto free_err;
 674                         }
 675
 676                         page = alloc_page(xs->sk.sk_allocation);
 677                         if (unlikely(!page)) {
 678                                 err = -EAGAIN;
 679                                 goto free_err;
 680                         }
 681
 682                         vaddr = kmap_local_page(page);
 683                         memcpy(vaddr, buffer, len);
 684                         kunmap_local(vaddr);
 685
 686                         skb_add_rx_frag(skb, nr_frags, page, 0, len, 0);
 687                 }
 688         }
 689
 690         skb->dev = dev;
 691         skb->priority = xs->sk.sk_priority;
 692         skb->mark = READ_ONCE(xs->sk.sk_mark);
 693         skb->destructor = xsk_destruct_skb;
 694         xsk_set_destructor_arg(skb);
 695
 696         return skb;
 697
 698 free_err:
 699         if (err == -EOVERFLOW) {
 700                 /* Drop the packet */
 701                 xsk_set_destructor_arg(xs->skb);
 702                 xsk_drop_skb(xs->skb);
 703                 xskq_cons_release(xs->tx);
 704         } else {
 705                 /* Let application retry */
 706                 xsk_cq_cancel_locked(xs, 1);
 707         }
 708
 709         return ERR_PTR(err);
 710 }
 711
 712 static int __xsk_generic_xmit(struct sock *sk)
 713 {
 714         struct xdp_sock *xs = xdp_sk(sk);
 715         u32 max_batch = TX_BATCH_SIZE;
 716         bool sent_frame = false;
 717         struct xdp_desc desc;
 718         struct sk_buff *skb;
 719         int err = 0;
 720
 721         mutex_lock(&xs->mutex);
 722
 723         /* Since we dropped the RCU read lock, the socket state might have changed. */
 724         if (unlikely(!xsk_is_bound(xs))) {
 725                 err = -ENXIO;
 726                 goto out;
 727         }
 728
 729         if (xs->queue_id >= xs->dev->real_num_tx_queues)
 730                 goto out;
 731
 732         while (xskq_cons_peek_desc(xs->tx, &desc, xs->pool)) {
 733                 if (max_batch-- == 0) {
 734                         err = -EAGAIN;
 735                         goto out;
 736                 }
 737
 738                 /* This is the backpressure mechanism for the Tx path.
 739                  * Reserve space in the completion queue and only proceed
 740                  * if there is space in it. This avoids having to implement
 741                  * any buffering in the Tx path.
 742                  */
 743                 if (xsk_cq_reserve_addr_locked(xs, desc.addr))
 744                         goto out;
 745
 746                 skb = xsk_build_skb(xs, &desc);
 747                 if (IS_ERR(skb)) {
 748                         err = PTR_ERR(skb);
 749                         if (err != -EOVERFLOW)
 750                                 goto out;
 751                         err = 0;
 752                         continue;
 753                 }
 754
 755                 xskq_cons_release(xs->tx);
 756
 757                 if (xp_mb_desc(&desc)) {
 758                         xs->skb = skb;
 759                         continue;
 760                 }
 761
 762                 err = __dev_direct_xmit(skb, xs->queue_id);
 763                 if  (err == NETDEV_TX_BUSY) {
 764                         /* Tell user-space to retry the send */
 765                         xskq_cons_cancel_n(xs->tx, xsk_get_num_desc(skb));
 766                         xsk_consume_skb(skb);
 767                         err = -EAGAIN;
 768                         goto out;
 769                 }
 770
 771                 /* Ignore NET_XMIT_CN as packet might have been sent */
 772                 if (err == NET_XMIT_DROP) {
 773                         /* SKB completed but not sent */
 774                         err = -EBUSY;
 775                         xs->skb = NULL;
 776                         goto out;
 777                 }
 778
 779                 sent_frame = true;
 780                 xs->skb = NULL;
 781         }
 782
 783         if (xskq_has_descs(xs->tx)) {
 784                 if (xs->skb)
 785                         xsk_drop_skb(xs->skb);
 786                 xskq_cons_release(xs->tx);
 787         }
 788
 789 out:
 790         if (sent_frame)
 791                 if (xsk_tx_writeable(xs))
 792                         sk->sk_write_space(sk);
 793
 794         mutex_unlock(&xs->mutex);
 795         return err;
 796 }
 797
 798 static int xsk_generic_xmit(struct sock *sk)
 799 {
 800         int ret;
 801
 802         /* Drop the RCU lock since the SKB path might sleep. */
 803         rcu_read_unlock();
 804         ret = __xsk_generic_xmit(sk);
 805         /* Reaquire RCU lock before going into common code. */
 806         rcu_read_lock();
 807
 808         return ret;
 809 }
 810
 811 static bool xsk_no_wakeup(struct sock *sk)
 812 {
 813 #ifdef CONFIG_NET_RX_BUSY_POLL
 814         /* Prefer busy-polling, skip the wakeup. */
 815         return READ_ONCE(sk->sk_prefer_busy_poll) && READ_ONCE(sk->sk_ll_usec) &&
 816                 READ_ONCE(sk->sk_napi_id) >= MIN_NAPI_ID;
 817 #else
 818         return false;
 819 #endif
 820 }
 821
 822 static int xsk_check_common(struct xdp_sock *xs)
 823 {
 824         if (unlikely(!xsk_is_bound(xs)))
 825                 return -ENXIO;
 826         if (unlikely(!(xs->dev->flags & IFF_UP)))
 827                 return -ENETDOWN;
 828
 829         return 0;
 830 }
 831
 832 static int __xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
 833 {
 834         bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
 835         struct sock *sk = sock->sk;
 836         struct xdp_sock *xs = xdp_sk(sk);
 837         struct xsk_buff_pool *pool;
 838         int err;
 839
 840         err = xsk_check_common(xs);
 841         if (err)
 842                 return err;
 843         if (unlikely(need_wait))
 844                 return -EOPNOTSUPP;
 845         if (unlikely(!xs->tx))
 846                 return -ENOBUFS;
 847
 848         if (sk_can_busy_loop(sk)) {
 849                 if (xs->zc)
 850                         __sk_mark_napi_id_once(sk, xsk_pool_get_napi_id(xs->pool));
 851                 sk_busy_loop(sk, 1); /* only support non-blocking sockets */
 852         }
 853
 854         if (xs->zc && xsk_no_wakeup(sk))
 855                 return 0;
 856
 857         pool = xs->pool;
 858         if (pool->cached_need_wakeup & XDP_WAKEUP_TX) {
 859                 if (xs->zc)
 860                         return xsk_wakeup(xs, XDP_WAKEUP_TX);
 861                 return xsk_generic_xmit(sk);
 862         }
 863         return 0;
 864 }
 865
 866 static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
 867 {
 868         int ret;
 869
 870         rcu_read_lock();
 871         ret = __xsk_sendmsg(sock, m, total_len);
 872         rcu_read_unlock();
 873
 874         return ret;
 875 }
 876
 877 static int __xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
 878 {
 879         bool need_wait = !(flags & MSG_DONTWAIT);
 880         struct sock *sk = sock->sk;
 881         struct xdp_sock *xs = xdp_sk(sk);
 882         int err;
 883
 884         err = xsk_check_common(xs);
 885         if (err)
 886                 return err;
 887         if (unlikely(!xs->rx))
 888                 return -ENOBUFS;
 889         if (unlikely(need_wait))
 890                 return -EOPNOTSUPP;
 891
 892         if (sk_can_busy_loop(sk))
 893                 sk_busy_loop(sk, 1); /* only support non-blocking sockets */
 894
 895         if (xsk_no_wakeup(sk))
 896                 return 0;
 897
 898         if (xs->pool->cached_need_wakeup & XDP_WAKEUP_RX && xs->zc)
 899                 return xsk_wakeup(xs, XDP_WAKEUP_RX);
 900         return 0;
 901 }
 902
 903 static int xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
 904 {
 905         int ret;
 906
 907         rcu_read_lock();
 908         ret = __xsk_recvmsg(sock, m, len, flags);
 909         rcu_read_unlock();
 910
 911         return ret;
 912 }
 913
 914 static __poll_t xsk_poll(struct file *file, struct socket *sock,
 915                              struct poll_table_struct *wait)
 916 {
 917         __poll_t mask = 0;
 918         struct sock *sk = sock->sk;
 919         struct xdp_sock *xs = xdp_sk(sk);
 920         struct xsk_buff_pool *pool;
 921
 922         sock_poll_wait(file, sock, wait);
 923
 924         rcu_read_lock();
 925         if (xsk_check_common(xs))
 926                 goto out;
 927
 928         pool = xs->pool;
 929
 930         if (pool->cached_need_wakeup) {
 931                 if (xs->zc)
 932                         xsk_wakeup(xs, pool->cached_need_wakeup);
 933                 else if (xs->tx)
 934                         /* Poll needs to drive Tx also in copy mode */
 935                         xsk_generic_xmit(sk);
 936         }
 937
 938         if (xs->rx && !xskq_prod_is_empty(xs->rx))
 939                 mask |= EPOLLIN | EPOLLRDNORM;
 940         if (xs->tx && xsk_tx_writeable(xs))
 941                 mask |= EPOLLOUT | EPOLLWRNORM;
 942 out:
 943         rcu_read_unlock();
 944         return mask;
 945 }
 946
 947 static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
 948                           bool umem_queue)
 949 {
 950         struct xsk_queue *q;
 951
 952         if (entries == 0 || *queue || !is_power_of_2(entries))
 953                 return -EINVAL;
 954
 955         q = xskq_create(entries, umem_queue);
 956         if (!q)
 957                 return -ENOMEM;
 958
 959         /* Make sure queue is ready before it can be seen by others */
 960         smp_wmb();
 961         WRITE_ONCE(*queue, q);
 962         return 0;
 963 }
 964
 965 static void xsk_unbind_dev(struct xdp_sock *xs)
 966 {
 967         struct net_device *dev = xs->dev;
 968
 969         if (xs->state != XSK_BOUND)
 970                 return;
 971         WRITE_ONCE(xs->state, XSK_UNBOUND);
 972
 973         /* Wait for driver to stop using the xdp socket. */
 974         xp_del_xsk(xs->pool, xs);
 975         synchronize_net();
 976         dev_put(dev);
 977 }
 978
 979 static struct xsk_map *xsk_get_map_list_entry(struct xdp_sock *xs,
 980                                               struct xdp_sock __rcu ***map_entry)
 981 {
 982         struct xsk_map *map = NULL;
 983         struct xsk_map_node *node;
 984
 985         *map_entry = NULL;
 986
 987         spin_lock_bh(&xs->map_list_lock);
 988         node = list_first_entry_or_null(&xs->map_list, struct xsk_map_node,
 989                                         node);
 990         if (node) {
 991                 bpf_map_inc(&node->map->map);
 992                 map = node->map;
 993                 *map_entry = node->map_entry;
 994         }
 995         spin_unlock_bh(&xs->map_list_lock);
 996         return map;
 997 }
 998
 999 static void xsk_delete_from_maps(struct xdp_sock *xs)
1000 {
1001         /* This function removes the current XDP socket from all the
1002          * maps it resides in. We need to take extra care here, due to
1003          * the two locks involved. Each map has a lock synchronizing
1004          * updates to the entries, and each socket has a lock that
1005          * synchronizes access to the list of maps (map_list). For
1006          * deadlock avoidance the locks need to be taken in the order
1007          * "map lock"->"socket map list lock". We start off by
1008          * accessing the socket map list, and take a reference to the
1009          * map to guarantee existence between the
1010          * xsk_get_map_list_entry() and xsk_map_try_sock_delete()
1011          * calls. Then we ask the map to remove the socket, which
1012          * tries to remove the socket from the map. Note that there
1013          * might be updates to the map between
1014          * xsk_get_map_list_entry() and xsk_map_try_sock_delete().
1015          */
1016         struct xdp_sock __rcu **map_entry = NULL;
1017         struct xsk_map *map;
1018
1019         while ((map = xsk_get_map_list_entry(xs, &map_entry))) {
1020                 xsk_map_try_sock_delete(map, xs, map_entry);
1021                 bpf_map_put(&map->map);
1022         }
1023 }
1024
1025 static int xsk_release(struct socket *sock)
1026 {
1027         struct sock *sk = sock->sk;
1028         struct xdp_sock *xs = xdp_sk(sk);
1029         struct net *net;
1030
1031         if (!sk)
1032                 return 0;
1033
1034         net = sock_net(sk);
1035
1036         if (xs->skb)
1037                 xsk_drop_skb(xs->skb);
1038
1039         mutex_lock(&net->xdp.lock);
1040         sk_del_node_init_rcu(sk);
1041         mutex_unlock(&net->xdp.lock);
1042
1043         sock_prot_inuse_add(net, sk->sk_prot, -1);
1044
1045         xsk_delete_from_maps(xs);
1046         mutex_lock(&xs->mutex);
1047         xsk_unbind_dev(xs);
1048         mutex_unlock(&xs->mutex);
1049
1050         xskq_destroy(xs->rx);
1051         xskq_destroy(xs->tx);
1052         xskq_destroy(xs->fq_tmp);
1053         xskq_destroy(xs->cq_tmp);
1054
1055         sock_orphan(sk);
1056         sock->sk = NULL;
1057
1058         sock_put(sk);
1059
1060         return 0;
1061 }
1062
1063 static struct socket *xsk_lookup_xsk_from_fd(int fd)
1064 {
1065         struct socket *sock;
1066         int err;
1067
1068         sock = sockfd_lookup(fd, &err);
1069         if (!sock)
1070                 return ERR_PTR(-ENOTSOCK);
1071
1072         if (sock->sk->sk_family != PF_XDP) {
1073                 sockfd_put(sock);
1074                 return ERR_PTR(-ENOPROTOOPT);
1075         }
1076
1077         return sock;
1078 }
1079
1080 static bool xsk_validate_queues(struct xdp_sock *xs)
1081 {
1082         return xs->fq_tmp && xs->cq_tmp;
1083 }
1084
1085 static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
1086 {
1087         struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
1088         struct sock *sk = sock->sk;
1089         struct xdp_sock *xs = xdp_sk(sk);
1090         struct net_device *dev;
1091         int bound_dev_if;
1092         u32 flags, qid;
1093         int err = 0;
1094
1095         if (addr_len < sizeof(struct sockaddr_xdp))
1096                 return -EINVAL;
1097         if (sxdp->sxdp_family != AF_XDP)
1098                 return -EINVAL;
1099
1100         flags = sxdp->sxdp_flags;
1101         if (flags & ~(XDP_SHARED_UMEM | XDP_COPY | XDP_ZEROCOPY |
1102                       XDP_USE_NEED_WAKEUP | XDP_USE_SG))
1103                 return -EINVAL;
1104
1105         bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1106         if (bound_dev_if && bound_dev_if != sxdp->sxdp_ifindex)
1107                 return -EINVAL;
1108
1109         rtnl_lock();
1110         mutex_lock(&xs->mutex);
1111         if (xs->state != XSK_READY) {
1112                 err = -EBUSY;
1113                 goto out_release;
1114         }
1115
1116         dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
1117         if (!dev) {
1118                 err = -ENODEV;
1119                 goto out_release;
1120         }
1121
1122         if (!xs->rx && !xs->tx) {
1123                 err = -EINVAL;
1124                 goto out_unlock;
1125         }
1126
1127         qid = sxdp->sxdp_queue_id;
1128
1129         if (flags & XDP_SHARED_UMEM) {
1130                 struct xdp_sock *umem_xs;
1131                 struct socket *sock;
1132
1133                 if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY) ||
1134                     (flags & XDP_USE_NEED_WAKEUP) || (flags & XDP_USE_SG)) {
1135                         /* Cannot specify flags for shared sockets. */
1136                         err = -EINVAL;
1137                         goto out_unlock;
1138                 }
1139
1140                 if (xs->umem) {
1141                         /* We have already our own. */
1142                         err = -EINVAL;
1143                         goto out_unlock;
1144                 }
1145
1146                 sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
1147                 if (IS_ERR(sock)) {
1148                         err = PTR_ERR(sock);
1149                         goto out_unlock;
1150                 }
1151
1152                 umem_xs = xdp_sk(sock->sk);
1153                 if (!xsk_is_bound(umem_xs)) {
1154                         err = -EBADF;
1155                         sockfd_put(sock);
1156                         goto out_unlock;
1157                 }
1158
1159                 if (umem_xs->queue_id != qid || umem_xs->dev != dev) {
1160                         /* Share the umem with another socket on another qid
1161                          * and/or device.
1162                          */
1163                         xs->pool = xp_create_and_assign_umem(xs,
1164                                                              umem_xs->umem);
1165                         if (!xs->pool) {
1166                                 err = -ENOMEM;
1167                                 sockfd_put(sock);
1168                                 goto out_unlock;
1169                         }
1170
1171                         err = xp_assign_dev_shared(xs->pool, umem_xs, dev,
1172                                                    qid);
1173                         if (err) {
1174                                 xp_destroy(xs->pool);
1175                                 xs->pool = NULL;
1176                                 sockfd_put(sock);
1177                                 goto out_unlock;
1178                         }
1179                 } else {
1180                         /* Share the buffer pool with the other socket. */
1181                         if (xs->fq_tmp || xs->cq_tmp) {
1182                                 /* Do not allow setting your own fq or cq. */
1183                                 err = -EINVAL;
1184                                 sockfd_put(sock);
1185                                 goto out_unlock;
1186                         }
1187
1188                         xp_get_pool(umem_xs->pool);
1189                         xs->pool = umem_xs->pool;
1190
1191                         /* If underlying shared umem was created without Tx
1192                          * ring, allocate Tx descs array that Tx batching API
1193                          * utilizes
1194                          */
1195                         if (xs->tx && !xs->pool->tx_descs) {
1196                                 err = xp_alloc_tx_descs(xs->pool, xs);
1197                                 if (err) {
1198                                         xp_put_pool(xs->pool);
1199                                         xs->pool = NULL;
1200                                         sockfd_put(sock);
1201                                         goto out_unlock;
1202                                 }
1203                         }
1204                 }
1205
1206                 xdp_get_umem(umem_xs->umem);
1207                 WRITE_ONCE(xs->umem, umem_xs->umem);
1208                 sockfd_put(sock);
1209         } else if (!xs->umem || !xsk_validate_queues(xs)) {
1210                 err = -EINVAL;
1211                 goto out_unlock;
1212         } else {
1213                 /* This xsk has its own umem. */
1214                 xs->pool = xp_create_and_assign_umem(xs, xs->umem);
1215                 if (!xs->pool) {
1216                         err = -ENOMEM;
1217                         goto out_unlock;
1218                 }
1219
1220                 err = xp_assign_dev(xs->pool, dev, qid, flags);
1221                 if (err) {
1222                         xp_destroy(xs->pool);
1223                         xs->pool = NULL;
1224                         goto out_unlock;
1225                 }
1226         }
1227
1228         /* FQ and CQ are now owned by the buffer pool and cleaned up with it. */
1229         xs->fq_tmp = NULL;
1230         xs->cq_tmp = NULL;
1231
1232         xs->dev = dev;
1233         xs->zc = xs->umem->zc;
1234         xs->sg = !!(xs->umem->flags & XDP_UMEM_SG_FLAG);
1235         xs->queue_id = qid;
1236         xp_add_xsk(xs->pool, xs);
1237
1238 out_unlock:
1239         if (err) {
1240                 dev_put(dev);
1241         } else {
1242                 /* Matches smp_rmb() in bind() for shared umem
1243                  * sockets, and xsk_is_bound().
1244                  */
1245                 smp_wmb();
1246                 WRITE_ONCE(xs->state, XSK_BOUND);
1247         }
1248 out_release:
1249         mutex_unlock(&xs->mutex);
1250         rtnl_unlock();
1251         return err;
1252 }
1253
1254 struct xdp_umem_reg_v1 {
1255         __u64 addr; /* Start of packet data area */
1256         __u64 len; /* Length of packet data area */
1257         __u32 chunk_size;
1258         __u32 headroom;
1259 };
1260
1261 static int xsk_setsockopt(struct socket *sock, int level, int optname,
1262                           sockptr_t optval, unsigned int optlen)
1263 {
1264         struct sock *sk = sock->sk;
1265         struct xdp_sock *xs = xdp_sk(sk);
1266         int err;
1267
1268         if (level != SOL_XDP)
1269                 return -ENOPROTOOPT;
1270
1271         switch (optname) {
1272         case XDP_RX_RING:
1273         case XDP_TX_RING:
1274         {
1275                 struct xsk_queue **q;
1276                 int entries;
1277
1278                 if (optlen < sizeof(entries))
1279                         return -EINVAL;
1280                 if (copy_from_sockptr(&entries, optval, sizeof(entries)))
1281                         return -EFAULT;
1282
1283                 mutex_lock(&xs->mutex);
1284                 if (xs->state != XSK_READY) {
1285                         mutex_unlock(&xs->mutex);
1286                         return -EBUSY;
1287                 }
1288                 q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
1289                 err = xsk_init_queue(entries, q, false);
1290                 if (!err && optname == XDP_TX_RING)
1291                         /* Tx needs to be explicitly woken up the first time */
1292                         xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
1293                 mutex_unlock(&xs->mutex);
1294                 return err;
1295         }
1296         case XDP_UMEM_REG:
1297         {
1298                 size_t mr_size = sizeof(struct xdp_umem_reg);
1299                 struct xdp_umem_reg mr = {};
1300                 struct xdp_umem *umem;
1301
1302                 if (optlen < sizeof(struct xdp_umem_reg_v1))
1303                         return -EINVAL;
1304                 else if (optlen < sizeof(mr))
1305                         mr_size = sizeof(struct xdp_umem_reg_v1);
1306
1307                 if (copy_from_sockptr(&mr, optval, mr_size))
1308                         return -EFAULT;
1309
1310                 mutex_lock(&xs->mutex);
1311                 if (xs->state != XSK_READY || xs->umem) {
1312                         mutex_unlock(&xs->mutex);
1313                         return -EBUSY;
1314                 }
1315
1316                 umem = xdp_umem_create(&mr);
1317                 if (IS_ERR(umem)) {
1318                         mutex_unlock(&xs->mutex);
1319                         return PTR_ERR(umem);
1320                 }
1321
1322                 /* Make sure umem is ready before it can be seen by others */
1323                 smp_wmb();
1324                 WRITE_ONCE(xs->umem, umem);
1325                 mutex_unlock(&xs->mutex);
1326                 return 0;
1327         }
1328         case XDP_UMEM_FILL_RING:
1329         case XDP_UMEM_COMPLETION_RING:
1330         {
1331                 struct xsk_queue **q;
1332                 int entries;
1333
1334                 if (copy_from_sockptr(&entries, optval, sizeof(entries)))
1335                         return -EFAULT;
1336
1337                 mutex_lock(&xs->mutex);
1338                 if (xs->state != XSK_READY) {
1339                         mutex_unlock(&xs->mutex);
1340                         return -EBUSY;
1341                 }
1342
1343                 q = (optname == XDP_UMEM_FILL_RING) ? &xs->fq_tmp :
1344                         &xs->cq_tmp;
1345                 err = xsk_init_queue(entries, q, true);
1346                 mutex_unlock(&xs->mutex);
1347                 return err;
1348         }
1349         default:
1350                 break;
1351         }
1352
1353         return -ENOPROTOOPT;
1354 }
1355
1356 static void xsk_enter_rxtx_offsets(struct xdp_ring_offset_v1 *ring)
1357 {
1358         ring->producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
1359         ring->consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
1360         ring->desc = offsetof(struct xdp_rxtx_ring, desc);
1361 }
1362
1363 static void xsk_enter_umem_offsets(struct xdp_ring_offset_v1 *ring)
1364 {
1365         ring->producer = offsetof(struct xdp_umem_ring, ptrs.producer);
1366         ring->consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
1367         ring->desc = offsetof(struct xdp_umem_ring, desc);
1368 }
1369
1370 struct xdp_statistics_v1 {
1371         __u64 rx_dropped;
1372         __u64 rx_invalid_descs;
1373         __u64 tx_invalid_descs;
1374 };
1375
1376 static int xsk_getsockopt(struct socket *sock, int level, int optname,
1377                           char __user *optval, int __user *optlen)
1378 {
1379         struct sock *sk = sock->sk;
1380         struct xdp_sock *xs = xdp_sk(sk);
1381         int len;
1382
1383         if (level != SOL_XDP)
1384                 return -ENOPROTOOPT;
1385
1386         if (get_user(len, optlen))
1387                 return -EFAULT;
1388         if (len < 0)
1389                 return -EINVAL;
1390
1391         switch (optname) {
1392         case XDP_STATISTICS:
1393         {
1394                 struct xdp_statistics stats = {};
1395                 bool extra_stats = true;
1396                 size_t stats_size;
1397
1398                 if (len < sizeof(struct xdp_statistics_v1)) {
1399                         return -EINVAL;
1400                 } else if (len < sizeof(stats)) {
1401                         extra_stats = false;
1402                         stats_size = sizeof(struct xdp_statistics_v1);
1403                 } else {
1404                         stats_size = sizeof(stats);
1405                 }
1406
1407                 mutex_lock(&xs->mutex);
1408                 stats.rx_dropped = xs->rx_dropped;
1409                 if (extra_stats) {
1410                         stats.rx_ring_full = xs->rx_queue_full;
1411                         stats.rx_fill_ring_empty_descs =
1412                                 xs->pool ? xskq_nb_queue_empty_descs(xs->pool->fq) : 0;
1413                         stats.tx_ring_empty_descs = xskq_nb_queue_empty_descs(xs->tx);
1414                 } else {
1415                         stats.rx_dropped += xs->rx_queue_full;
1416                 }
1417                 stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
1418                 stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
1419                 mutex_unlock(&xs->mutex);
1420
1421                 if (copy_to_user(optval, &stats, stats_size))
1422                         return -EFAULT;
1423                 if (put_user(stats_size, optlen))
1424                         return -EFAULT;
1425
1426                 return 0;
1427         }
1428         case XDP_MMAP_OFFSETS:
1429         {
1430                 struct xdp_mmap_offsets off;
1431                 struct xdp_mmap_offsets_v1 off_v1;
1432                 bool flags_supported = true;
1433                 void *to_copy;
1434
1435                 if (len < sizeof(off_v1))
1436                         return -EINVAL;
1437                 else if (len < sizeof(off))
1438                         flags_supported = false;
1439
1440                 if (flags_supported) {
1441                         /* xdp_ring_offset is identical to xdp_ring_offset_v1
1442                          * except for the flags field added to the end.
1443                          */
1444                         xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
1445                                                &off.rx);
1446                         xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
1447                                                &off.tx);
1448                         xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
1449                                                &off.fr);
1450                         xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
1451                                                &off.cr);
1452                         off.rx.flags = offsetof(struct xdp_rxtx_ring,
1453                                                 ptrs.flags);
1454                         off.tx.flags = offsetof(struct xdp_rxtx_ring,
1455                                                 ptrs.flags);
1456                         off.fr.flags = offsetof(struct xdp_umem_ring,
1457                                                 ptrs.flags);
1458                         off.cr.flags = offsetof(struct xdp_umem_ring,
1459                                                 ptrs.flags);
1460
1461                         len = sizeof(off);
1462                         to_copy = &off;
1463                 } else {
1464                         xsk_enter_rxtx_offsets(&off_v1.rx);
1465                         xsk_enter_rxtx_offsets(&off_v1.tx);
1466                         xsk_enter_umem_offsets(&off_v1.fr);
1467                         xsk_enter_umem_offsets(&off_v1.cr);
1468
1469                         len = sizeof(off_v1);
1470                         to_copy = &off_v1;
1471                 }
1472
1473                 if (copy_to_user(optval, to_copy, len))
1474                         return -EFAULT;
1475                 if (put_user(len, optlen))
1476                         return -EFAULT;
1477
1478                 return 0;
1479         }
1480         case XDP_OPTIONS:
1481         {
1482                 struct xdp_options opts = {};
1483
1484                 if (len < sizeof(opts))
1485                         return -EINVAL;
1486
1487                 mutex_lock(&xs->mutex);
1488                 if (xs->zc)
1489                         opts.flags |= XDP_OPTIONS_ZEROCOPY;
1490                 mutex_unlock(&xs->mutex);
1491
1492                 len = sizeof(opts);
1493                 if (copy_to_user(optval, &opts, len))
1494                         return -EFAULT;
1495                 if (put_user(len, optlen))
1496                         return -EFAULT;
1497
1498                 return 0;
1499         }
1500         default:
1501                 break;
1502         }
1503
1504         return -EOPNOTSUPP;
1505 }
1506
1507 static int xsk_mmap(struct file *file, struct socket *sock,
1508                     struct vm_area_struct *vma)
1509 {
1510         loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1511         unsigned long size = vma->vm_end - vma->vm_start;
1512         struct xdp_sock *xs = xdp_sk(sock->sk);
1513         int state = READ_ONCE(xs->state);
1514         struct xsk_queue *q = NULL;
1515
1516         if (state != XSK_READY && state != XSK_BOUND)
1517                 return -EBUSY;
1518
1519         if (offset == XDP_PGOFF_RX_RING) {
1520                 q = READ_ONCE(xs->rx);
1521         } else if (offset == XDP_PGOFF_TX_RING) {
1522                 q = READ_ONCE(xs->tx);
1523         } else {
1524                 /* Matches the smp_wmb() in XDP_UMEM_REG */
1525                 smp_rmb();
1526                 if (offset == XDP_UMEM_PGOFF_FILL_RING)
1527                         q = state == XSK_READY ? READ_ONCE(xs->fq_tmp) :
1528                                                  READ_ONCE(xs->pool->fq);
1529                 else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
1530                         q = state == XSK_READY ? READ_ONCE(xs->cq_tmp) :
1531                                                  READ_ONCE(xs->pool->cq);
1532         }
1533
1534         if (!q)
1535                 return -EINVAL;
1536
1537         /* Matches the smp_wmb() in xsk_init_queue */
1538         smp_rmb();
1539         if (size > q->ring_vmalloc_size)
1540                 return -EINVAL;
1541
1542         return remap_vmalloc_range(vma, q->ring, 0);
1543 }
1544
1545 static int xsk_notifier(struct notifier_block *this,
1546                         unsigned long msg, void *ptr)
1547 {
1548         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1549         struct net *net = dev_net(dev);
1550         struct sock *sk;
1551
1552         switch (msg) {
1553         case NETDEV_UNREGISTER:
1554                 mutex_lock(&net->xdp.lock);
1555                 sk_for_each(sk, &net->xdp.list) {
1556                         struct xdp_sock *xs = xdp_sk(sk);
1557
1558                         mutex_lock(&xs->mutex);
1559                         if (xs->dev == dev) {
1560                                 sk->sk_err = ENETDOWN;
1561                                 if (!sock_flag(sk, SOCK_DEAD))
1562                                         sk_error_report(sk);
1563
1564                                 xsk_unbind_dev(xs);
1565
1566                                 /* Clear device references. */
1567                                 xp_clear_dev(xs->pool);
1568                         }
1569                         mutex_unlock(&xs->mutex);
1570                 }
1571                 mutex_unlock(&net->xdp.lock);
1572                 break;
1573         }
1574         return NOTIFY_DONE;
1575 }
1576
1577 static struct proto xsk_proto = {
1578         .name =         "XDP",
1579         .owner =        THIS_MODULE,
1580         .obj_size =     sizeof(struct xdp_sock),
1581 };
1582
1583 static const struct proto_ops xsk_proto_ops = {
1584         .family         = PF_XDP,
1585         .owner          = THIS_MODULE,
1586         .release        = xsk_release,
1587         .bind           = xsk_bind,
1588         .connect        = sock_no_connect,
1589         .socketpair     = sock_no_socketpair,
1590         .accept         = sock_no_accept,
1591         .getname        = sock_no_getname,
1592         .poll           = xsk_poll,
1593         .ioctl          = sock_no_ioctl,
1594         .listen         = sock_no_listen,
1595         .shutdown       = sock_no_shutdown,
1596         .setsockopt     = xsk_setsockopt,
1597         .getsockopt     = xsk_getsockopt,
1598         .sendmsg        = xsk_sendmsg,
1599         .recvmsg        = xsk_recvmsg,
1600         .mmap           = xsk_mmap,
1601 };
1602
1603 static void xsk_destruct(struct sock *sk)
1604 {
1605         struct xdp_sock *xs = xdp_sk(sk);
1606
1607         if (!sock_flag(sk, SOCK_DEAD))
1608                 return;
1609
1610         if (!xp_put_pool(xs->pool))
1611                 xdp_put_umem(xs->umem, !xs->pool);
1612 }
1613
1614 static int xsk_create(struct net *net, struct socket *sock, int protocol,
1615                       int kern)
1616 {
1617         struct xdp_sock *xs;
1618         struct sock *sk;
1619
1620         if (!ns_capable(net->user_ns, CAP_NET_RAW))
1621                 return -EPERM;
1622         if (sock->type != SOCK_RAW)
1623                 return -ESOCKTNOSUPPORT;
1624
1625         if (protocol)
1626                 return -EPROTONOSUPPORT;
1627
1628         sock->state = SS_UNCONNECTED;
1629
1630         sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
1631         if (!sk)
1632                 return -ENOBUFS;
1633
1634         sock->ops = &xsk_proto_ops;
1635
1636         sock_init_data(sock, sk);
1637
1638         sk->sk_family = PF_XDP;
1639
1640         sk->sk_destruct = xsk_destruct;
1641
1642         sock_set_flag(sk, SOCK_RCU_FREE);
1643
1644         xs = xdp_sk(sk);
1645         xs->state = XSK_READY;
1646         mutex_init(&xs->mutex);
1647         spin_lock_init(&xs->rx_lock);
1648
1649         INIT_LIST_HEAD(&xs->map_list);
1650         spin_lock_init(&xs->map_list_lock);
1651
1652         mutex_lock(&net->xdp.lock);
1653         sk_add_node_rcu(sk, &net->xdp.list);
1654         mutex_unlock(&net->xdp.lock);
1655
1656         sock_prot_inuse_add(net, &xsk_proto, 1);
1657
1658         return 0;
1659 }
1660
1661 static const struct net_proto_family xsk_family_ops = {
1662         .family = PF_XDP,
1663         .create = xsk_create,
1664         .owner  = THIS_MODULE,
1665 };
1666
1667 static struct notifier_block xsk_netdev_notifier = {
1668         .notifier_call  = xsk_notifier,
1669 };
1670
1671 static int __net_init xsk_net_init(struct net *net)
1672 {
1673         mutex_init(&net->xdp.lock);
1674         INIT_HLIST_HEAD(&net->xdp.list);
1675         return 0;
1676 }
1677
1678 static void __net_exit xsk_net_exit(struct net *net)
1679 {
1680         WARN_ON_ONCE(!hlist_empty(&net->xdp.list));
1681 }
1682
1683 static struct pernet_operations xsk_net_ops = {
1684         .init = xsk_net_init,
1685         .exit = xsk_net_exit,
1686 };
1687
1688 static int __init xsk_init(void)
1689 {
1690         int err, cpu;
1691
1692         err = proto_register(&xsk_proto, 0 /* no slab */);
1693         if (err)
1694                 goto out;
1695
1696         err = sock_register(&xsk_family_ops);
1697         if (err)
1698                 goto out_proto;
1699
1700         err = register_pernet_subsys(&xsk_net_ops);
1701         if (err)
1702                 goto out_sk;
1703
1704         err = register_netdevice_notifier(&xsk_netdev_notifier);
1705         if (err)
1706                 goto out_pernet;
1707
1708         for_each_possible_cpu(cpu)
1709                 INIT_LIST_HEAD(&per_cpu(xskmap_flush_list, cpu));
1710         return 0;
1711
1712 out_pernet:
1713         unregister_pernet_subsys(&xsk_net_ops);
1714 out_sk:
1715         sock_unregister(PF_XDP);
1716 out_proto:
1717         proto_unregister(&xsk_proto);
1718 out:
1719         return err;
1720 }
1721
1722 fs_initcall(xsk_init);