net/xdp/xsk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* XDP sockets
   3  *
   4  * AF_XDP sockets allows a channel between XDP programs and userspace
   5  * applications.
   6  * Copyright(c) 2018 Intel Corporation.
   7  *
   8  * Author(s): Björn Töpel <bjorn.topel@intel.com>
   9  *            Magnus Karlsson <magnus.karlsson@intel.com>
  10  */
  11
  12 #define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
  13
  14 #include <linux/if_xdp.h>
  15 #include <linux/init.h>
  16 #include <linux/sched/mm.h>
  17 #include <linux/sched/signal.h>
  18 #include <linux/sched/task.h>
  19 #include <linux/socket.h>
  20 #include <linux/file.h>
  21 #include <linux/uaccess.h>
  22 #include <linux/net.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/rculist.h>
  25 #include <linux/vmalloc.h>
  26 #include <net/xdp_sock_drv.h>
  27 #include <net/busy_poll.h>
  28 #include <net/xdp.h>
  29
  30 #include "xsk_queue.h"
  31 #include "xdp_umem.h"
  32 #include "xsk.h"
  33
  34 #define TX_BATCH_SIZE 32
  35
  36 static DEFINE_PER_CPU(struct list_head, xskmap_flush_list);
  37
  38 void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
  39 {
  40         if (pool->cached_need_wakeup & XDP_WAKEUP_RX)
  41                 return;
  42
  43         pool->fq->ring->flags |= XDP_RING_NEED_WAKEUP;
  44         pool->cached_need_wakeup |= XDP_WAKEUP_RX;
  45 }
  46 EXPORT_SYMBOL(xsk_set_rx_need_wakeup);
  47
  48 void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
  49 {
  50         struct xdp_sock *xs;
  51
  52         if (pool->cached_need_wakeup & XDP_WAKEUP_TX)
  53                 return;
  54
  55         rcu_read_lock();
  56         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
  57                 xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
  58         }
  59         rcu_read_unlock();
  60
  61         pool->cached_need_wakeup |= XDP_WAKEUP_TX;
  62 }
  63 EXPORT_SYMBOL(xsk_set_tx_need_wakeup);
  64
  65 void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
  66 {
  67         if (!(pool->cached_need_wakeup & XDP_WAKEUP_RX))
  68                 return;
  69
  70         pool->fq->ring->flags &= ~XDP_RING_NEED_WAKEUP;
  71         pool->cached_need_wakeup &= ~XDP_WAKEUP_RX;
  72 }
  73 EXPORT_SYMBOL(xsk_clear_rx_need_wakeup);
  74
  75 void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
  76 {
  77         struct xdp_sock *xs;
  78
  79         if (!(pool->cached_need_wakeup & XDP_WAKEUP_TX))
  80                 return;
  81
  82         rcu_read_lock();
  83         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
  84                 xs->tx->ring->flags &= ~XDP_RING_NEED_WAKEUP;
  85         }
  86         rcu_read_unlock();
  87
  88         pool->cached_need_wakeup &= ~XDP_WAKEUP_TX;
  89 }
  90 EXPORT_SYMBOL(xsk_clear_tx_need_wakeup);
  91
  92 bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
  93 {
  94         return pool->uses_need_wakeup;
  95 }
  96 EXPORT_SYMBOL(xsk_uses_need_wakeup);
  97
  98 struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
  99                                             u16 queue_id)
 100 {
 101         if (queue_id < dev->real_num_rx_queues)
 102                 return dev->_rx[queue_id].pool;
 103         if (queue_id < dev->real_num_tx_queues)
 104                 return dev->_tx[queue_id].pool;
 105
 106         return NULL;
 107 }
 108 EXPORT_SYMBOL(xsk_get_pool_from_qid);
 109
 110 void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id)
 111 {
 112         if (queue_id < dev->num_rx_queues)
 113                 dev->_rx[queue_id].pool = NULL;
 114         if (queue_id < dev->num_tx_queues)
 115                 dev->_tx[queue_id].pool = NULL;
 116 }
 117
 118 /* The buffer pool is stored both in the _rx struct and the _tx struct as we do
 119  * not know if the device has more tx queues than rx, or the opposite.
 120  * This might also change during run time.
 121  */
 122 int xsk_reg_pool_at_qid(struct net_device *dev, struct xsk_buff_pool *pool,
 123                         u16 queue_id)
 124 {
 125         if (queue_id >= max_t(unsigned int,
 126                               dev->real_num_rx_queues,
 127                               dev->real_num_tx_queues))
 128                 return -EINVAL;
 129
 130         if (queue_id < dev->real_num_rx_queues)
 131                 dev->_rx[queue_id].pool = pool;
 132         if (queue_id < dev->real_num_tx_queues)
 133                 dev->_tx[queue_id].pool = pool;
 134
 135         return 0;
 136 }
 137
 138 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len,
 139                         u32 flags)
 140 {
 141         u64 addr;
 142         int err;
 143
 144         addr = xp_get_handle(xskb);
 145         err = xskq_prod_reserve_desc(xs->rx, addr, len, flags);
 146         if (err) {
 147                 xs->rx_queue_full++;
 148                 return err;
 149         }
 150
 151         xp_release(xskb);
 152         return 0;
 153 }
 154
 155 static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 156 {
 157         struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
 158         u32 frags = xdp_buff_has_frags(xdp);
 159         struct xdp_buff_xsk *pos, *tmp;
 160         struct list_head *xskb_list;
 161         u32 contd = 0;
 162         int err;
 163
 164         if (frags)
 165                 contd = XDP_PKT_CONTD;
 166
 167         err = __xsk_rcv_zc(xs, xskb, len, contd);
 168         if (err || likely(!frags))
 169                 goto out;
 170
 171         xskb_list = &xskb->pool->xskb_list;
 172         list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) {
 173                 if (list_is_singular(xskb_list))
 174                         contd = 0;
 175                 len = pos->xdp.data_end - pos->xdp.data;
 176                 err = __xsk_rcv_zc(xs, pos, len, contd);
 177                 if (err)
 178                         return err;
 179                 list_del(&pos->xskb_list_node);
 180         }
 181
 182 out:
 183         return err;
 184 }
 185
 186 static void *xsk_copy_xdp_start(struct xdp_buff *from)
 187 {
 188         if (unlikely(xdp_data_meta_unsupported(from)))
 189                 return from->data;
 190         else
 191                 return from->data_meta;
 192 }
 193
 194 static u32 xsk_copy_xdp(void *to, void **from, u32 to_len,
 195                         u32 *from_len, skb_frag_t **frag, u32 rem)
 196 {
 197         u32 copied = 0;
 198
 199         while (1) {
 200                 u32 copy_len = min_t(u32, *from_len, to_len);
 201
 202                 memcpy(to, *from, copy_len);
 203                 copied += copy_len;
 204                 if (rem == copied)
 205                         return copied;
 206
 207                 if (*from_len == copy_len) {
 208                         *from = skb_frag_address(*frag);
 209                         *from_len = skb_frag_size((*frag)++);
 210                 } else {
 211                         *from += copy_len;
 212                         *from_len -= copy_len;
 213                 }
 214                 if (to_len == copy_len)
 215                         return copied;
 216
 217                 to_len -= copy_len;
 218                 to += copy_len;
 219         }
 220 }
 221
 222 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 223 {
 224         u32 frame_size = xsk_pool_get_rx_frame_size(xs->pool);
 225         void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
 226         u32 from_len, meta_len, rem, num_desc;
 227         struct xdp_buff_xsk *xskb;
 228         struct xdp_buff *xsk_xdp;
 229         skb_frag_t *frag;
 230
 231         from_len = xdp->data_end - copy_from;
 232         meta_len = xdp->data - copy_from;
 233         rem = len + meta_len;
 234
 235         if (len <= frame_size && !xdp_buff_has_frags(xdp)) {
 236                 int err;
 237
 238                 xsk_xdp = xsk_buff_alloc(xs->pool);
 239                 if (!xsk_xdp) {
 240                         xs->rx_dropped++;
 241                         return -ENOMEM;
 242                 }
 243                 memcpy(xsk_xdp->data - meta_len, copy_from, rem);
 244                 xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 245                 err = __xsk_rcv_zc(xs, xskb, len, 0);
 246                 if (err) {
 247                         xsk_buff_free(xsk_xdp);
 248                         return err;
 249                 }
 250
 251                 return 0;
 252         }
 253
 254         num_desc = (len - 1) / frame_size + 1;
 255
 256         if (!xsk_buff_can_alloc(xs->pool, num_desc)) {
 257                 xs->rx_dropped++;
 258                 return -ENOMEM;
 259         }
 260         if (xskq_prod_nb_free(xs->rx, num_desc) < num_desc) {
 261                 xs->rx_queue_full++;
 262                 return -ENOBUFS;
 263         }
 264
 265         if (xdp_buff_has_frags(xdp)) {
 266                 struct skb_shared_info *sinfo;
 267
 268                 sinfo = xdp_get_shared_info_from_buff(xdp);
 269                 frag =  &sinfo->frags[0];
 270         }
 271
 272         do {
 273                 u32 to_len = frame_size + meta_len;
 274                 u32 copied;
 275
 276                 xsk_xdp = xsk_buff_alloc(xs->pool);
 277                 copy_to = xsk_xdp->data - meta_len;
 278
 279                 copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
 280                 rem -= copied;
 281
 282                 xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 283                 __xsk_rcv_zc(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0);
 284                 meta_len = 0;
 285         } while (rem);
 286
 287         return 0;
 288 }
 289
 290 static bool xsk_tx_writeable(struct xdp_sock *xs)
 291 {
 292         if (xskq_cons_present_entries(xs->tx) > xs->tx->nentries / 2)
 293                 return false;
 294
 295         return true;
 296 }
 297
 298 static bool xsk_is_bound(struct xdp_sock *xs)
 299 {
 300         if (READ_ONCE(xs->state) == XSK_BOUND) {
 301                 /* Matches smp_wmb() in bind(). */
 302                 smp_rmb();
 303                 return true;
 304         }
 305         return false;
 306 }
 307
 308 static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 309 {
 310         if (!xsk_is_bound(xs))
 311                 return -ENXIO;
 312
 313         if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
 314                 return -EINVAL;
 315
 316         if (len > xsk_pool_get_rx_frame_size(xs->pool) && !xs->sg) {
 317                 xs->rx_dropped++;
 318                 return -ENOSPC;
 319         }
 320
 321         sk_mark_napi_id_once_xdp(&xs->sk, xdp);
 322         return 0;
 323 }
 324
 325 static void xsk_flush(struct xdp_sock *xs)
 326 {
 327         xskq_prod_submit(xs->rx);
 328         __xskq_cons_release(xs->pool->fq);
 329         sock_def_readable(&xs->sk);
 330 }
 331
 332 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
 333 {
 334         u32 len = xdp_get_buff_len(xdp);
 335         int err;
 336
 337         spin_lock_bh(&xs->rx_lock);
 338         err = xsk_rcv_check(xs, xdp, len);
 339         if (!err) {
 340                 err = __xsk_rcv(xs, xdp, len);
 341                 xsk_flush(xs);
 342         }
 343         spin_unlock_bh(&xs->rx_lock);
 344         return err;
 345 }
 346
 347 static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
 348 {
 349         u32 len = xdp_get_buff_len(xdp);
 350         int err;
 351
 352         err = xsk_rcv_check(xs, xdp, len);
 353         if (err)
 354                 return err;
 355
 356         if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
 357                 len = xdp->data_end - xdp->data;
 358                 return xsk_rcv_zc(xs, xdp, len);
 359         }
 360
 361         err = __xsk_rcv(xs, xdp, len);
 362         if (!err)
 363                 xdp_return_buff(xdp);
 364         return err;
 365 }
 366
 367 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
 368 {
 369         struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
 370         int err;
 371
 372         err = xsk_rcv(xs, xdp);
 373         if (err)
 374                 return err;
 375
 376         if (!xs->flush_node.prev)
 377                 list_add(&xs->flush_node, flush_list);
 378
 379         return 0;
 380 }
 381
 382 void __xsk_map_flush(void)
 383 {
 384         struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
 385         struct xdp_sock *xs, *tmp;
 386
 387         list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
 388                 xsk_flush(xs);
 389                 __list_del_clearprev(&xs->flush_node);
 390         }
 391 }
 392
 393 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 394 {
 395         xskq_prod_submit_n(pool->cq, nb_entries);
 396 }
 397 EXPORT_SYMBOL(xsk_tx_completed);
 398
 399 void xsk_tx_release(struct xsk_buff_pool *pool)
 400 {
 401         struct xdp_sock *xs;
 402
 403         rcu_read_lock();
 404         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
 405                 __xskq_cons_release(xs->tx);
 406                 if (xsk_tx_writeable(xs))
 407                         xs->sk.sk_write_space(&xs->sk);
 408         }
 409         rcu_read_unlock();
 410 }
 411 EXPORT_SYMBOL(xsk_tx_release);
 412
 413 bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
 414 {
 415         struct xdp_sock *xs;
 416
 417         rcu_read_lock();
 418         list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
 419                 if (!xskq_cons_peek_desc(xs->tx, desc, pool)) {
 420                         if (xskq_has_descs(xs->tx))
 421                                 xskq_cons_release(xs->tx);
 422                         continue;
 423                 }
 424
 425                 /* This is the backpressure mechanism for the Tx path.
 426                  * Reserve space in the completion queue and only proceed
 427                  * if there is space in it. This avoids having to implement
 428                  * any buffering in the Tx path.
 429                  */
 430                 if (xskq_prod_reserve_addr(pool->cq, desc->addr))
 431                         goto out;
 432
 433                 xskq_cons_release(xs->tx);
 434                 rcu_read_unlock();
 435                 return true;
 436         }
 437
 438 out:
 439         rcu_read_unlock();
 440         return false;
 441 }
 442 EXPORT_SYMBOL(xsk_tx_peek_desc);
 443
 444 static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries)
 445 {
 446         struct xdp_desc *descs = pool->tx_descs;
 447         u32 nb_pkts = 0;
 448
 449         while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts]))
 450                 nb_pkts++;
 451
 452         xsk_tx_release(pool);
 453         return nb_pkts;
 454 }
 455
 456 u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts)
 457 {
 458         struct xdp_sock *xs;
 459
 460         rcu_read_lock();
 461         if (!list_is_singular(&pool->xsk_tx_list)) {
 462                 /* Fallback to the non-batched version */
 463                 rcu_read_unlock();
 464                 return xsk_tx_peek_release_fallback(pool, nb_pkts);
 465         }
 466
 467         xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
 468         if (!xs) {
 469                 nb_pkts = 0;
 470                 goto out;
 471         }
 472
 473         nb_pkts = xskq_cons_nb_entries(xs->tx, nb_pkts);
 474
 475         /* This is the backpressure mechanism for the Tx path. Try to
 476          * reserve space in the completion queue for all packets, but
 477          * if there are fewer slots available, just process that many
 478          * packets. This avoids having to implement any buffering in
 479          * the Tx path.
 480          */
 481         nb_pkts = xskq_prod_nb_free(pool->cq, nb_pkts);
 482         if (!nb_pkts)
 483                 goto out;
 484
 485         nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, nb_pkts);
 486         if (!nb_pkts) {
 487                 xs->tx->queue_empty_descs++;
 488                 goto out;
 489         }
 490
 491         __xskq_cons_release(xs->tx);
 492         xskq_prod_write_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
 493         xs->sk.sk_write_space(&xs->sk);
 494
 495 out:
 496         rcu_read_unlock();
 497         return nb_pkts;
 498 }
 499 EXPORT_SYMBOL(xsk_tx_peek_release_desc_batch);
 500
 501 static int xsk_wakeup(struct xdp_sock *xs, u8 flags)
 502 {
 503         struct net_device *dev = xs->dev;
 504
 505         return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
 506 }
 507
 508 static int xsk_cq_reserve_addr_locked(struct xdp_sock *xs, u64 addr)
 509 {
 510         unsigned long flags;
 511         int ret;
 512
 513         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 514         ret = xskq_prod_reserve_addr(xs->pool->cq, addr);
 515         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 516
 517         return ret;
 518 }
 519
 520 static void xsk_cq_submit_locked(struct xdp_sock *xs, u32 n)
 521 {
 522         unsigned long flags;
 523
 524         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 525         xskq_prod_submit_n(xs->pool->cq, n);
 526         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 527 }
 528
 529 static void xsk_cq_cancel_locked(struct xdp_sock *xs, u32 n)
 530 {
 531         unsigned long flags;
 532
 533         spin_lock_irqsave(&xs->pool->cq_lock, flags);
 534         xskq_prod_cancel_n(xs->pool->cq, n);
 535         spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 536 }
 537
 538 static u32 xsk_get_num_desc(struct sk_buff *skb)
 539 {
 540         return skb ? (long)skb_shinfo(skb)->destructor_arg : 0;
 541 }
 542
 543 static void xsk_destruct_skb(struct sk_buff *skb)
 544 {
 545         xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb));
 546         sock_wfree(skb);
 547 }
 548
 549 static void xsk_set_destructor_arg(struct sk_buff *skb)
 550 {
 551         long num = xsk_get_num_desc(xdp_sk(skb->sk)->skb) + 1;
 552
 553         skb_shinfo(skb)->destructor_arg = (void *)num;
 554 }
 555
 556 static void xsk_consume_skb(struct sk_buff *skb)
 557 {
 558         struct xdp_sock *xs = xdp_sk(skb->sk);
 559
 560         skb->destructor = sock_wfree;
 561         xsk_cq_cancel_locked(xs, xsk_get_num_desc(skb));
 562         /* Free skb without triggering the perf drop trace */
 563         consume_skb(skb);
 564         xs->skb = NULL;
 565 }
 566
 567 static void xsk_drop_skb(struct sk_buff *skb)
 568 {
 569         xdp_sk(skb->sk)->tx->invalid_descs += xsk_get_num_desc(skb);
 570         xsk_consume_skb(skb);
 571 }
 572
 573 static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
 574                                               struct xdp_desc *desc)
 575 {
 576         struct xsk_buff_pool *pool = xs->pool;
 577         u32 hr, len, ts, offset, copy, copied;
 578         struct sk_buff *skb = xs->skb;
 579         struct page *page;
 580         void *buffer;
 581         int err, i;
 582         u64 addr;
 583
 584         if (!skb) {
 585                 hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom));
 586
 587                 skb = sock_alloc_send_skb(&xs->sk, hr, 1, &err);
 588                 if (unlikely(!skb))
 589                         return ERR_PTR(err);
 590
 591                 skb_reserve(skb, hr);
 592         }
 593
 594         addr = desc->addr;
 595         len = desc->len;
 596         ts = pool->unaligned ? len : pool->chunk_size;
 597
 598         buffer = xsk_buff_raw_get_data(pool, addr);
 599         offset = offset_in_page(buffer);
 600         addr = buffer - pool->addrs;
 601
 602         for (copied = 0, i = skb_shinfo(skb)->nr_frags; copied < len; i++) {
 603                 if (unlikely(i >= MAX_SKB_FRAGS))
 604                         return ERR_PTR(-EFAULT);
 605
 606                 page = pool->umem->pgs[addr >> PAGE_SHIFT];
 607                 get_page(page);
 608
 609                 copy = min_t(u32, PAGE_SIZE - offset, len - copied);
 610                 skb_fill_page_desc(skb, i, page, offset, copy);
 611
 612                 copied += copy;
 613                 addr += copy;
 614                 offset = 0;
 615         }
 616
 617         skb->len += len;
 618         skb->data_len += len;
 619         skb->truesize += ts;
 620
 621         refcount_add(ts, &xs->sk.sk_wmem_alloc);
 622
 623         return skb;
 624 }
 625
 626 static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
 627                                      struct xdp_desc *desc)
 628 {
 629         struct net_device *dev = xs->dev;
 630         struct sk_buff *skb = xs->skb;
 631         int err;
 632
 633         if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {
 634                 skb = xsk_build_skb_zerocopy(xs, desc);
 635                 if (IS_ERR(skb)) {
 636                         err = PTR_ERR(skb);
 637                         goto free_err;
 638                 }
 639         } else {
 640                 u32 hr, tr, len;
 641                 void *buffer;
 642
 643                 buffer = xsk_buff_raw_get_data(xs->pool, desc->addr);
 644                 len = desc->len;
 645
 646                 if (!skb) {
 647                         hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom));
 648                         tr = dev->needed_tailroom;
 649                         skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err);
 650                         if (unlikely(!skb))
 651                                 goto free_err;
 652
 653                         skb_reserve(skb, hr);
 654                         skb_put(skb, len);
 655
 656                         err = skb_store_bits(skb, 0, buffer, len);
 657                         if (unlikely(err))
 658                                 goto free_err;
 659                 } else {
 660                         int nr_frags = skb_shinfo(skb)->nr_frags;
 661                         struct page *page;
 662                         u8 *vaddr;
 663
 664                         if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) {
 665                                 err = -EFAULT;
 666                                 goto free_err;
 667                         }
 668
 669                         page = alloc_page(xs->sk.sk_allocation);
 670                         if (unlikely(!page)) {
 671                                 err = -EAGAIN;
 672                                 goto free_err;
 673                         }
 674
 675                         vaddr = kmap_local_page(page);
 676                         memcpy(vaddr, buffer, len);
 677                         kunmap_local(vaddr);
 678
 679                         skb_add_rx_frag(skb, nr_frags, page, 0, len, 0);
 680                 }
 681         }
 682
 683         skb->dev = dev;
 684         skb->priority = xs->sk.sk_priority;
 685         skb->mark = xs->sk.sk_mark;
 686         skb->destructor = xsk_destruct_skb;
 687         xsk_set_destructor_arg(skb);
 688
 689         return skb;
 690
 691 free_err:
 692         if (err == -EAGAIN) {
 693                 xsk_cq_cancel_locked(xs, 1);
 694         } else {
 695                 xsk_set_destructor_arg(skb);
 696                 xsk_drop_skb(skb);
 697                 xskq_cons_release(xs->tx);
 698         }
 699
 700         return ERR_PTR(err);
 701 }
 702
 703 static int __xsk_generic_xmit(struct sock *sk)
 704 {
 705         struct xdp_sock *xs = xdp_sk(sk);
 706         u32 max_batch = TX_BATCH_SIZE;
 707         bool sent_frame = false;
 708         struct xdp_desc desc;
 709         struct sk_buff *skb;
 710         int err = 0;
 711
 712         mutex_lock(&xs->mutex);
 713
 714         /* Since we dropped the RCU read lock, the socket state might have changed. */
 715         if (unlikely(!xsk_is_bound(xs))) {
 716                 err = -ENXIO;
 717                 goto out;
 718         }
 719
 720         if (xs->queue_id >= xs->dev->real_num_tx_queues)
 721                 goto out;
 722
 723         while (xskq_cons_peek_desc(xs->tx, &desc, xs->pool)) {
 724                 if (max_batch-- == 0) {
 725                         err = -EAGAIN;
 726                         goto out;
 727                 }
 728
 729                 /* This is the backpressure mechanism for the Tx path.
 730                  * Reserve space in the completion queue and only proceed
 731                  * if there is space in it. This avoids having to implement
 732                  * any buffering in the Tx path.
 733                  */
 734                 if (xsk_cq_reserve_addr_locked(xs, desc.addr))
 735                         goto out;
 736
 737                 skb = xsk_build_skb(xs, &desc);
 738                 if (IS_ERR(skb)) {
 739                         err = PTR_ERR(skb);
 740                         if (err == -EAGAIN)
 741                                 goto out;
 742                         err = 0;
 743                         continue;
 744                 }
 745
 746                 xskq_cons_release(xs->tx);
 747
 748                 if (xp_mb_desc(&desc)) {
 749                         xs->skb = skb;
 750                         continue;
 751                 }
 752
 753                 err = __dev_direct_xmit(skb, xs->queue_id);
 754                 if  (err == NETDEV_TX_BUSY) {
 755                         /* Tell user-space to retry the send */
 756                         xskq_cons_cancel_n(xs->tx, xsk_get_num_desc(skb));
 757                         xsk_consume_skb(skb);
 758                         err = -EAGAIN;
 759                         goto out;
 760                 }
 761
 762                 /* Ignore NET_XMIT_CN as packet might have been sent */
 763                 if (err == NET_XMIT_DROP) {
 764                         /* SKB completed but not sent */
 765                         err = -EBUSY;
 766                         xs->skb = NULL;
 767                         goto out;
 768                 }
 769
 770                 sent_frame = true;
 771                 xs->skb = NULL;
 772         }
 773
 774         if (xskq_has_descs(xs->tx)) {
 775                 if (xs->skb)
 776                         xsk_drop_skb(xs->skb);
 777                 xskq_cons_release(xs->tx);
 778         }
 779
 780 out:
 781         if (sent_frame)
 782                 if (xsk_tx_writeable(xs))
 783                         sk->sk_write_space(sk);
 784
 785         mutex_unlock(&xs->mutex);
 786         return err;
 787 }
 788
 789 static int xsk_generic_xmit(struct sock *sk)
 790 {
 791         int ret;
 792
 793         /* Drop the RCU lock since the SKB path might sleep. */
 794         rcu_read_unlock();
 795         ret = __xsk_generic_xmit(sk);
 796         /* Reaquire RCU lock before going into common code. */
 797         rcu_read_lock();
 798
 799         return ret;
 800 }
 801
 802 static bool xsk_no_wakeup(struct sock *sk)
 803 {
 804 #ifdef CONFIG_NET_RX_BUSY_POLL
 805         /* Prefer busy-polling, skip the wakeup. */
 806         return READ_ONCE(sk->sk_prefer_busy_poll) && READ_ONCE(sk->sk_ll_usec) &&
 807                 READ_ONCE(sk->sk_napi_id) >= MIN_NAPI_ID;
 808 #else
 809         return false;
 810 #endif
 811 }
 812
 813 static int xsk_check_common(struct xdp_sock *xs)
 814 {
 815         if (unlikely(!xsk_is_bound(xs)))
 816                 return -ENXIO;
 817         if (unlikely(!(xs->dev->flags & IFF_UP)))
 818                 return -ENETDOWN;
 819
 820         return 0;
 821 }
 822
 823 static int __xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
 824 {
 825         bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
 826         struct sock *sk = sock->sk;
 827         struct xdp_sock *xs = xdp_sk(sk);
 828         struct xsk_buff_pool *pool;
 829         int err;
 830
 831         err = xsk_check_common(xs);
 832         if (err)
 833                 return err;
 834         if (unlikely(need_wait))
 835                 return -EOPNOTSUPP;
 836         if (unlikely(!xs->tx))
 837                 return -ENOBUFS;
 838
 839         if (sk_can_busy_loop(sk)) {
 840                 if (xs->zc)
 841                         __sk_mark_napi_id_once(sk, xsk_pool_get_napi_id(xs->pool));
 842                 sk_busy_loop(sk, 1); /* only support non-blocking sockets */
 843         }
 844
 845         if (xs->zc && xsk_no_wakeup(sk))
 846                 return 0;
 847
 848         pool = xs->pool;
 849         if (pool->cached_need_wakeup & XDP_WAKEUP_TX) {
 850                 if (xs->zc)
 851                         return xsk_wakeup(xs, XDP_WAKEUP_TX);
 852                 return xsk_generic_xmit(sk);
 853         }
 854         return 0;
 855 }
 856
 857 static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
 858 {
 859         int ret;
 860
 861         rcu_read_lock();
 862         ret = __xsk_sendmsg(sock, m, total_len);
 863         rcu_read_unlock();
 864
 865         return ret;
 866 }
 867
 868 static int __xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
 869 {
 870         bool need_wait = !(flags & MSG_DONTWAIT);
 871         struct sock *sk = sock->sk;
 872         struct xdp_sock *xs = xdp_sk(sk);
 873         int err;
 874
 875         err = xsk_check_common(xs);
 876         if (err)
 877                 return err;
 878         if (unlikely(!xs->rx))
 879                 return -ENOBUFS;
 880         if (unlikely(need_wait))
 881                 return -EOPNOTSUPP;
 882
 883         if (sk_can_busy_loop(sk))
 884                 sk_busy_loop(sk, 1); /* only support non-blocking sockets */
 885
 886         if (xsk_no_wakeup(sk))
 887                 return 0;
 888
 889         if (xs->pool->cached_need_wakeup & XDP_WAKEUP_RX && xs->zc)
 890                 return xsk_wakeup(xs, XDP_WAKEUP_RX);
 891         return 0;
 892 }
 893
 894 static int xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
 895 {
 896         int ret;
 897
 898         rcu_read_lock();
 899         ret = __xsk_recvmsg(sock, m, len, flags);
 900         rcu_read_unlock();
 901
 902         return ret;
 903 }
 904
 905 static __poll_t xsk_poll(struct file *file, struct socket *sock,
 906                              struct poll_table_struct *wait)
 907 {
 908         __poll_t mask = 0;
 909         struct sock *sk = sock->sk;
 910         struct xdp_sock *xs = xdp_sk(sk);
 911         struct xsk_buff_pool *pool;
 912
 913         sock_poll_wait(file, sock, wait);
 914
 915         rcu_read_lock();
 916         if (xsk_check_common(xs))
 917                 goto skip_tx;
 918
 919         pool = xs->pool;
 920
 921         if (pool->cached_need_wakeup) {
 922                 if (xs->zc)
 923                         xsk_wakeup(xs, pool->cached_need_wakeup);
 924                 else if (xs->tx)
 925                         /* Poll needs to drive Tx also in copy mode */
 926                         xsk_generic_xmit(sk);
 927         }
 928
 929 skip_tx:
 930         if (xs->rx && !xskq_prod_is_empty(xs->rx))
 931                 mask |= EPOLLIN | EPOLLRDNORM;
 932         if (xs->tx && xsk_tx_writeable(xs))
 933                 mask |= EPOLLOUT | EPOLLWRNORM;
 934
 935         rcu_read_unlock();
 936         return mask;
 937 }
 938
 939 static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
 940                           bool umem_queue)
 941 {
 942         struct xsk_queue *q;
 943
 944         if (entries == 0 || *queue || !is_power_of_2(entries))
 945                 return -EINVAL;
 946
 947         q = xskq_create(entries, umem_queue);
 948         if (!q)
 949                 return -ENOMEM;
 950
 951         /* Make sure queue is ready before it can be seen by others */
 952         smp_wmb();
 953         WRITE_ONCE(*queue, q);
 954         return 0;
 955 }
 956
 957 static void xsk_unbind_dev(struct xdp_sock *xs)
 958 {
 959         struct net_device *dev = xs->dev;
 960
 961         if (xs->state != XSK_BOUND)
 962                 return;
 963         WRITE_ONCE(xs->state, XSK_UNBOUND);
 964
 965         /* Wait for driver to stop using the xdp socket. */
 966         xp_del_xsk(xs->pool, xs);
 967         synchronize_net();
 968         dev_put(dev);
 969 }
 970
 971 static struct xsk_map *xsk_get_map_list_entry(struct xdp_sock *xs,
 972                                               struct xdp_sock __rcu ***map_entry)
 973 {
 974         struct xsk_map *map = NULL;
 975         struct xsk_map_node *node;
 976
 977         *map_entry = NULL;
 978
 979         spin_lock_bh(&xs->map_list_lock);
 980         node = list_first_entry_or_null(&xs->map_list, struct xsk_map_node,
 981                                         node);
 982         if (node) {
 983                 bpf_map_inc(&node->map->map);
 984                 map = node->map;
 985                 *map_entry = node->map_entry;
 986         }
 987         spin_unlock_bh(&xs->map_list_lock);
 988         return map;
 989 }
 990
 991 static void xsk_delete_from_maps(struct xdp_sock *xs)
 992 {
 993         /* This function removes the current XDP socket from all the
 994          * maps it resides in. We need to take extra care here, due to
 995          * the two locks involved. Each map has a lock synchronizing
 996          * updates to the entries, and each socket has a lock that
 997          * synchronizes access to the list of maps (map_list). For
 998          * deadlock avoidance the locks need to be taken in the order
 999          * "map lock"->"socket map list lock". We start off by
1000          * accessing the socket map list, and take a reference to the
1001          * map to guarantee existence between the
1002          * xsk_get_map_list_entry() and xsk_map_try_sock_delete()
1003          * calls. Then we ask the map to remove the socket, which
1004          * tries to remove the socket from the map. Note that there
1005          * might be updates to the map between
1006          * xsk_get_map_list_entry() and xsk_map_try_sock_delete().
1007          */
1008         struct xdp_sock __rcu **map_entry = NULL;
1009         struct xsk_map *map;
1010
1011         while ((map = xsk_get_map_list_entry(xs, &map_entry))) {
1012                 xsk_map_try_sock_delete(map, xs, map_entry);
1013                 bpf_map_put(&map->map);
1014         }
1015 }
1016
1017 static int xsk_release(struct socket *sock)
1018 {
1019         struct sock *sk = sock->sk;
1020         struct xdp_sock *xs = xdp_sk(sk);
1021         struct net *net;
1022
1023         if (!sk)
1024                 return 0;
1025
1026         net = sock_net(sk);
1027
1028         if (xs->skb)
1029                 xsk_drop_skb(xs->skb);
1030
1031         mutex_lock(&net->xdp.lock);
1032         sk_del_node_init_rcu(sk);
1033         mutex_unlock(&net->xdp.lock);
1034
1035         sock_prot_inuse_add(net, sk->sk_prot, -1);
1036
1037         xsk_delete_from_maps(xs);
1038         mutex_lock(&xs->mutex);
1039         xsk_unbind_dev(xs);
1040         mutex_unlock(&xs->mutex);
1041
1042         xskq_destroy(xs->rx);
1043         xskq_destroy(xs->tx);
1044         xskq_destroy(xs->fq_tmp);
1045         xskq_destroy(xs->cq_tmp);
1046
1047         sock_orphan(sk);
1048         sock->sk = NULL;
1049
1050         sock_put(sk);
1051
1052         return 0;
1053 }
1054
1055 static struct socket *xsk_lookup_xsk_from_fd(int fd)
1056 {
1057         struct socket *sock;
1058         int err;
1059
1060         sock = sockfd_lookup(fd, &err);
1061         if (!sock)
1062                 return ERR_PTR(-ENOTSOCK);
1063
1064         if (sock->sk->sk_family != PF_XDP) {
1065                 sockfd_put(sock);
1066                 return ERR_PTR(-ENOPROTOOPT);
1067         }
1068
1069         return sock;
1070 }
1071
1072 static bool xsk_validate_queues(struct xdp_sock *xs)
1073 {
1074         return xs->fq_tmp && xs->cq_tmp;
1075 }
1076
1077 static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
1078 {
1079         struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
1080         struct sock *sk = sock->sk;
1081         struct xdp_sock *xs = xdp_sk(sk);
1082         struct net_device *dev;
1083         int bound_dev_if;
1084         u32 flags, qid;
1085         int err = 0;
1086
1087         if (addr_len < sizeof(struct sockaddr_xdp))
1088                 return -EINVAL;
1089         if (sxdp->sxdp_family != AF_XDP)
1090                 return -EINVAL;
1091
1092         flags = sxdp->sxdp_flags;
1093         if (flags & ~(XDP_SHARED_UMEM | XDP_COPY | XDP_ZEROCOPY |
1094                       XDP_USE_NEED_WAKEUP | XDP_USE_SG))
1095                 return -EINVAL;
1096
1097         bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1098         if (bound_dev_if && bound_dev_if != sxdp->sxdp_ifindex)
1099                 return -EINVAL;
1100
1101         rtnl_lock();
1102         mutex_lock(&xs->mutex);
1103         if (xs->state != XSK_READY) {
1104                 err = -EBUSY;
1105                 goto out_release;
1106         }
1107
1108         dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex);
1109         if (!dev) {
1110                 err = -ENODEV;
1111                 goto out_release;
1112         }
1113
1114         if (!xs->rx && !xs->tx) {
1115                 err = -EINVAL;
1116                 goto out_unlock;
1117         }
1118
1119         qid = sxdp->sxdp_queue_id;
1120
1121         if (flags & XDP_SHARED_UMEM) {
1122                 struct xdp_sock *umem_xs;
1123                 struct socket *sock;
1124
1125                 if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY) ||
1126                     (flags & XDP_USE_NEED_WAKEUP) || (flags & XDP_USE_SG)) {
1127                         /* Cannot specify flags for shared sockets. */
1128                         err = -EINVAL;
1129                         goto out_unlock;
1130                 }
1131
1132                 if (xs->umem) {
1133                         /* We have already our own. */
1134                         err = -EINVAL;
1135                         goto out_unlock;
1136                 }
1137
1138                 sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);
1139                 if (IS_ERR(sock)) {
1140                         err = PTR_ERR(sock);
1141                         goto out_unlock;
1142                 }
1143
1144                 umem_xs = xdp_sk(sock->sk);
1145                 if (!xsk_is_bound(umem_xs)) {
1146                         err = -EBADF;
1147                         sockfd_put(sock);
1148                         goto out_unlock;
1149                 }
1150
1151                 if (umem_xs->queue_id != qid || umem_xs->dev != dev) {
1152                         /* Share the umem with another socket on another qid
1153                          * and/or device.
1154                          */
1155                         xs->pool = xp_create_and_assign_umem(xs,
1156                                                              umem_xs->umem);
1157                         if (!xs->pool) {
1158                                 err = -ENOMEM;
1159                                 sockfd_put(sock);
1160                                 goto out_unlock;
1161                         }
1162
1163                         err = xp_assign_dev_shared(xs->pool, umem_xs, dev,
1164                                                    qid);
1165                         if (err) {
1166                                 xp_destroy(xs->pool);
1167                                 xs->pool = NULL;
1168                                 sockfd_put(sock);
1169                                 goto out_unlock;
1170                         }
1171                 } else {
1172                         /* Share the buffer pool with the other socket. */
1173                         if (xs->fq_tmp || xs->cq_tmp) {
1174                                 /* Do not allow setting your own fq or cq. */
1175                                 err = -EINVAL;
1176                                 sockfd_put(sock);
1177                                 goto out_unlock;
1178                         }
1179
1180                         xp_get_pool(umem_xs->pool);
1181                         xs->pool = umem_xs->pool;
1182
1183                         /* If underlying shared umem was created without Tx
1184                          * ring, allocate Tx descs array that Tx batching API
1185                          * utilizes
1186                          */
1187                         if (xs->tx && !xs->pool->tx_descs) {
1188                                 err = xp_alloc_tx_descs(xs->pool, xs);
1189                                 if (err) {
1190                                         xp_put_pool(xs->pool);
1191                                         sockfd_put(sock);
1192                                         goto out_unlock;
1193                                 }
1194                         }
1195                 }
1196
1197                 xdp_get_umem(umem_xs->umem);
1198                 WRITE_ONCE(xs->umem, umem_xs->umem);
1199                 sockfd_put(sock);
1200         } else if (!xs->umem || !xsk_validate_queues(xs)) {
1201                 err = -EINVAL;
1202                 goto out_unlock;
1203         } else {
1204                 /* This xsk has its own umem. */
1205                 xs->pool = xp_create_and_assign_umem(xs, xs->umem);
1206                 if (!xs->pool) {
1207                         err = -ENOMEM;
1208                         goto out_unlock;
1209                 }
1210
1211                 err = xp_assign_dev(xs->pool, dev, qid, flags);
1212                 if (err) {
1213                         xp_destroy(xs->pool);
1214                         xs->pool = NULL;
1215                         goto out_unlock;
1216                 }
1217         }
1218
1219         /* FQ and CQ are now owned by the buffer pool and cleaned up with it. */
1220         xs->fq_tmp = NULL;
1221         xs->cq_tmp = NULL;
1222
1223         xs->dev = dev;
1224         xs->zc = xs->umem->zc;
1225         xs->sg = !!(flags & XDP_USE_SG);
1226         xs->queue_id = qid;
1227         xp_add_xsk(xs->pool, xs);
1228
1229 out_unlock:
1230         if (err) {
1231                 dev_put(dev);
1232         } else {
1233                 /* Matches smp_rmb() in bind() for shared umem
1234                  * sockets, and xsk_is_bound().
1235                  */
1236                 smp_wmb();
1237                 WRITE_ONCE(xs->state, XSK_BOUND);
1238         }
1239 out_release:
1240         mutex_unlock(&xs->mutex);
1241         rtnl_unlock();
1242         return err;
1243 }
1244
1245 struct xdp_umem_reg_v1 {
1246         __u64 addr; /* Start of packet data area */
1247         __u64 len; /* Length of packet data area */
1248         __u32 chunk_size;
1249         __u32 headroom;
1250 };
1251
1252 static int xsk_setsockopt(struct socket *sock, int level, int optname,
1253                           sockptr_t optval, unsigned int optlen)
1254 {
1255         struct sock *sk = sock->sk;
1256         struct xdp_sock *xs = xdp_sk(sk);
1257         int err;
1258
1259         if (level != SOL_XDP)
1260                 return -ENOPROTOOPT;
1261
1262         switch (optname) {
1263         case XDP_RX_RING:
1264         case XDP_TX_RING:
1265         {
1266                 struct xsk_queue **q;
1267                 int entries;
1268
1269                 if (optlen < sizeof(entries))
1270                         return -EINVAL;
1271                 if (copy_from_sockptr(&entries, optval, sizeof(entries)))
1272                         return -EFAULT;
1273
1274                 mutex_lock(&xs->mutex);
1275                 if (xs->state != XSK_READY) {
1276                         mutex_unlock(&xs->mutex);
1277                         return -EBUSY;
1278                 }
1279                 q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
1280                 err = xsk_init_queue(entries, q, false);
1281                 if (!err && optname == XDP_TX_RING)
1282                         /* Tx needs to be explicitly woken up the first time */
1283                         xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
1284                 mutex_unlock(&xs->mutex);
1285                 return err;
1286         }
1287         case XDP_UMEM_REG:
1288         {
1289                 size_t mr_size = sizeof(struct xdp_umem_reg);
1290                 struct xdp_umem_reg mr = {};
1291                 struct xdp_umem *umem;
1292
1293                 if (optlen < sizeof(struct xdp_umem_reg_v1))
1294                         return -EINVAL;
1295                 else if (optlen < sizeof(mr))
1296                         mr_size = sizeof(struct xdp_umem_reg_v1);
1297
1298                 if (copy_from_sockptr(&mr, optval, mr_size))
1299                         return -EFAULT;
1300
1301                 mutex_lock(&xs->mutex);
1302                 if (xs->state != XSK_READY || xs->umem) {
1303                         mutex_unlock(&xs->mutex);
1304                         return -EBUSY;
1305                 }
1306
1307                 umem = xdp_umem_create(&mr);
1308                 if (IS_ERR(umem)) {
1309                         mutex_unlock(&xs->mutex);
1310                         return PTR_ERR(umem);
1311                 }
1312
1313                 /* Make sure umem is ready before it can be seen by others */
1314                 smp_wmb();
1315                 WRITE_ONCE(xs->umem, umem);
1316                 mutex_unlock(&xs->mutex);
1317                 return 0;
1318         }
1319         case XDP_UMEM_FILL_RING:
1320         case XDP_UMEM_COMPLETION_RING:
1321         {
1322                 struct xsk_queue **q;
1323                 int entries;
1324
1325                 if (copy_from_sockptr(&entries, optval, sizeof(entries)))
1326                         return -EFAULT;
1327
1328                 mutex_lock(&xs->mutex);
1329                 if (xs->state != XSK_READY) {
1330                         mutex_unlock(&xs->mutex);
1331                         return -EBUSY;
1332                 }
1333
1334                 q = (optname == XDP_UMEM_FILL_RING) ? &xs->fq_tmp :
1335                         &xs->cq_tmp;
1336                 err = xsk_init_queue(entries, q, true);
1337                 mutex_unlock(&xs->mutex);
1338                 return err;
1339         }
1340         default:
1341                 break;
1342         }
1343
1344         return -ENOPROTOOPT;
1345 }
1346
1347 static void xsk_enter_rxtx_offsets(struct xdp_ring_offset_v1 *ring)
1348 {
1349         ring->producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
1350         ring->consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
1351         ring->desc = offsetof(struct xdp_rxtx_ring, desc);
1352 }
1353
1354 static void xsk_enter_umem_offsets(struct xdp_ring_offset_v1 *ring)
1355 {
1356         ring->producer = offsetof(struct xdp_umem_ring, ptrs.producer);
1357         ring->consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
1358         ring->desc = offsetof(struct xdp_umem_ring, desc);
1359 }
1360
1361 struct xdp_statistics_v1 {
1362         __u64 rx_dropped;
1363         __u64 rx_invalid_descs;
1364         __u64 tx_invalid_descs;
1365 };
1366
1367 static int xsk_getsockopt(struct socket *sock, int level, int optname,
1368                           char __user *optval, int __user *optlen)
1369 {
1370         struct sock *sk = sock->sk;
1371         struct xdp_sock *xs = xdp_sk(sk);
1372         int len;
1373
1374         if (level != SOL_XDP)
1375                 return -ENOPROTOOPT;
1376
1377         if (get_user(len, optlen))
1378                 return -EFAULT;
1379         if (len < 0)
1380                 return -EINVAL;
1381
1382         switch (optname) {
1383         case XDP_STATISTICS:
1384         {
1385                 struct xdp_statistics stats = {};
1386                 bool extra_stats = true;
1387                 size_t stats_size;
1388
1389                 if (len < sizeof(struct xdp_statistics_v1)) {
1390                         return -EINVAL;
1391                 } else if (len < sizeof(stats)) {
1392                         extra_stats = false;
1393                         stats_size = sizeof(struct xdp_statistics_v1);
1394                 } else {
1395                         stats_size = sizeof(stats);
1396                 }
1397
1398                 mutex_lock(&xs->mutex);
1399                 stats.rx_dropped = xs->rx_dropped;
1400                 if (extra_stats) {
1401                         stats.rx_ring_full = xs->rx_queue_full;
1402                         stats.rx_fill_ring_empty_descs =
1403                                 xs->pool ? xskq_nb_queue_empty_descs(xs->pool->fq) : 0;
1404                         stats.tx_ring_empty_descs = xskq_nb_queue_empty_descs(xs->tx);
1405                 } else {
1406                         stats.rx_dropped += xs->rx_queue_full;
1407                 }
1408                 stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx);
1409                 stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx);
1410                 mutex_unlock(&xs->mutex);
1411
1412                 if (copy_to_user(optval, &stats, stats_size))
1413                         return -EFAULT;
1414                 if (put_user(stats_size, optlen))
1415                         return -EFAULT;
1416
1417                 return 0;
1418         }
1419         case XDP_MMAP_OFFSETS:
1420         {
1421                 struct xdp_mmap_offsets off;
1422                 struct xdp_mmap_offsets_v1 off_v1;
1423                 bool flags_supported = true;
1424                 void *to_copy;
1425
1426                 if (len < sizeof(off_v1))
1427                         return -EINVAL;
1428                 else if (len < sizeof(off))
1429                         flags_supported = false;
1430
1431                 if (flags_supported) {
1432                         /* xdp_ring_offset is identical to xdp_ring_offset_v1
1433                          * except for the flags field added to the end.
1434                          */
1435                         xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
1436                                                &off.rx);
1437                         xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *)
1438                                                &off.tx);
1439                         xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
1440                                                &off.fr);
1441                         xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *)
1442                                                &off.cr);
1443                         off.rx.flags = offsetof(struct xdp_rxtx_ring,
1444                                                 ptrs.flags);
1445                         off.tx.flags = offsetof(struct xdp_rxtx_ring,
1446                                                 ptrs.flags);
1447                         off.fr.flags = offsetof(struct xdp_umem_ring,
1448                                                 ptrs.flags);
1449                         off.cr.flags = offsetof(struct xdp_umem_ring,
1450                                                 ptrs.flags);
1451
1452                         len = sizeof(off);
1453                         to_copy = &off;
1454                 } else {
1455                         xsk_enter_rxtx_offsets(&off_v1.rx);
1456                         xsk_enter_rxtx_offsets(&off_v1.tx);
1457                         xsk_enter_umem_offsets(&off_v1.fr);
1458                         xsk_enter_umem_offsets(&off_v1.cr);
1459
1460                         len = sizeof(off_v1);
1461                         to_copy = &off_v1;
1462                 }
1463
1464                 if (copy_to_user(optval, to_copy, len))
1465                         return -EFAULT;
1466                 if (put_user(len, optlen))
1467                         return -EFAULT;
1468
1469                 return 0;
1470         }
1471         case XDP_OPTIONS:
1472         {
1473                 struct xdp_options opts = {};
1474
1475                 if (len < sizeof(opts))
1476                         return -EINVAL;
1477
1478                 mutex_lock(&xs->mutex);
1479                 if (xs->zc)
1480                         opts.flags |= XDP_OPTIONS_ZEROCOPY;
1481                 mutex_unlock(&xs->mutex);
1482
1483                 len = sizeof(opts);
1484                 if (copy_to_user(optval, &opts, len))
1485                         return -EFAULT;
1486                 if (put_user(len, optlen))
1487                         return -EFAULT;
1488
1489                 return 0;
1490         }
1491         default:
1492                 break;
1493         }
1494
1495         return -EOPNOTSUPP;
1496 }
1497
1498 static int xsk_mmap(struct file *file, struct socket *sock,
1499                     struct vm_area_struct *vma)
1500 {
1501         loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1502         unsigned long size = vma->vm_end - vma->vm_start;
1503         struct xdp_sock *xs = xdp_sk(sock->sk);
1504         int state = READ_ONCE(xs->state);
1505         struct xsk_queue *q = NULL;
1506
1507         if (state != XSK_READY && state != XSK_BOUND)
1508                 return -EBUSY;
1509
1510         if (offset == XDP_PGOFF_RX_RING) {
1511                 q = READ_ONCE(xs->rx);
1512         } else if (offset == XDP_PGOFF_TX_RING) {
1513                 q = READ_ONCE(xs->tx);
1514         } else {
1515                 /* Matches the smp_wmb() in XDP_UMEM_REG */
1516                 smp_rmb();
1517                 if (offset == XDP_UMEM_PGOFF_FILL_RING)
1518                         q = state == XSK_READY ? READ_ONCE(xs->fq_tmp) :
1519                                                  READ_ONCE(xs->pool->fq);
1520                 else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
1521                         q = state == XSK_READY ? READ_ONCE(xs->cq_tmp) :
1522                                                  READ_ONCE(xs->pool->cq);
1523         }
1524
1525         if (!q)
1526                 return -EINVAL;
1527
1528         /* Matches the smp_wmb() in xsk_init_queue */
1529         smp_rmb();
1530         if (size > q->ring_vmalloc_size)
1531                 return -EINVAL;
1532
1533         return remap_vmalloc_range(vma, q->ring, 0);
1534 }
1535
1536 static int xsk_notifier(struct notifier_block *this,
1537                         unsigned long msg, void *ptr)
1538 {
1539         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1540         struct net *net = dev_net(dev);
1541         struct sock *sk;
1542
1543         switch (msg) {
1544         case NETDEV_UNREGISTER:
1545                 mutex_lock(&net->xdp.lock);
1546                 sk_for_each(sk, &net->xdp.list) {
1547                         struct xdp_sock *xs = xdp_sk(sk);
1548
1549                         mutex_lock(&xs->mutex);
1550                         if (xs->dev == dev) {
1551                                 sk->sk_err = ENETDOWN;
1552                                 if (!sock_flag(sk, SOCK_DEAD))
1553                                         sk_error_report(sk);
1554
1555                                 xsk_unbind_dev(xs);
1556
1557                                 /* Clear device references. */
1558                                 xp_clear_dev(xs->pool);
1559                         }
1560                         mutex_unlock(&xs->mutex);
1561                 }
1562                 mutex_unlock(&net->xdp.lock);
1563                 break;
1564         }
1565         return NOTIFY_DONE;
1566 }
1567
1568 static struct proto xsk_proto = {
1569         .name =         "XDP",
1570         .owner =        THIS_MODULE,
1571         .obj_size =     sizeof(struct xdp_sock),
1572 };
1573
1574 static const struct proto_ops xsk_proto_ops = {
1575         .family         = PF_XDP,
1576         .owner          = THIS_MODULE,
1577         .release        = xsk_release,
1578         .bind           = xsk_bind,
1579         .connect        = sock_no_connect,
1580         .socketpair     = sock_no_socketpair,
1581         .accept         = sock_no_accept,
1582         .getname        = sock_no_getname,
1583         .poll           = xsk_poll,
1584         .ioctl          = sock_no_ioctl,
1585         .listen         = sock_no_listen,
1586         .shutdown       = sock_no_shutdown,
1587         .setsockopt     = xsk_setsockopt,
1588         .getsockopt     = xsk_getsockopt,
1589         .sendmsg        = xsk_sendmsg,
1590         .recvmsg        = xsk_recvmsg,
1591         .mmap           = xsk_mmap,
1592 };
1593
1594 static void xsk_destruct(struct sock *sk)
1595 {
1596         struct xdp_sock *xs = xdp_sk(sk);
1597
1598         if (!sock_flag(sk, SOCK_DEAD))
1599                 return;
1600
1601         if (!xp_put_pool(xs->pool))
1602                 xdp_put_umem(xs->umem, !xs->pool);
1603 }
1604
1605 static int xsk_create(struct net *net, struct socket *sock, int protocol,
1606                       int kern)
1607 {
1608         struct xdp_sock *xs;
1609         struct sock *sk;
1610
1611         if (!ns_capable(net->user_ns, CAP_NET_RAW))
1612                 return -EPERM;
1613         if (sock->type != SOCK_RAW)
1614                 return -ESOCKTNOSUPPORT;
1615
1616         if (protocol)
1617                 return -EPROTONOSUPPORT;
1618
1619         sock->state = SS_UNCONNECTED;
1620
1621         sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern);
1622         if (!sk)
1623                 return -ENOBUFS;
1624
1625         sock->ops = &xsk_proto_ops;
1626
1627         sock_init_data(sock, sk);
1628
1629         sk->sk_family = PF_XDP;
1630
1631         sk->sk_destruct = xsk_destruct;
1632
1633         sock_set_flag(sk, SOCK_RCU_FREE);
1634
1635         xs = xdp_sk(sk);
1636         xs->state = XSK_READY;
1637         mutex_init(&xs->mutex);
1638         spin_lock_init(&xs->rx_lock);
1639
1640         INIT_LIST_HEAD(&xs->map_list);
1641         spin_lock_init(&xs->map_list_lock);
1642
1643         mutex_lock(&net->xdp.lock);
1644         sk_add_node_rcu(sk, &net->xdp.list);
1645         mutex_unlock(&net->xdp.lock);
1646
1647         sock_prot_inuse_add(net, &xsk_proto, 1);
1648
1649         return 0;
1650 }
1651
1652 static const struct net_proto_family xsk_family_ops = {
1653         .family = PF_XDP,
1654         .create = xsk_create,
1655         .owner  = THIS_MODULE,
1656 };
1657
1658 static struct notifier_block xsk_netdev_notifier = {
1659         .notifier_call  = xsk_notifier,
1660 };
1661
1662 static int __net_init xsk_net_init(struct net *net)
1663 {
1664         mutex_init(&net->xdp.lock);
1665         INIT_HLIST_HEAD(&net->xdp.list);
1666         return 0;
1667 }
1668
1669 static void __net_exit xsk_net_exit(struct net *net)
1670 {
1671         WARN_ON_ONCE(!hlist_empty(&net->xdp.list));
1672 }
1673
1674 static struct pernet_operations xsk_net_ops = {
1675         .init = xsk_net_init,
1676         .exit = xsk_net_exit,
1677 };
1678
1679 static int __init xsk_init(void)
1680 {
1681         int err, cpu;
1682
1683         err = proto_register(&xsk_proto, 0 /* no slab */);
1684         if (err)
1685                 goto out;
1686
1687         err = sock_register(&xsk_family_ops);
1688         if (err)
1689                 goto out_proto;
1690
1691         err = register_pernet_subsys(&xsk_net_ops);
1692         if (err)
1693                 goto out_sk;
1694
1695         err = register_netdevice_notifier(&xsk_netdev_notifier);
1696         if (err)
1697                 goto out_pernet;
1698
1699         for_each_possible_cpu(cpu)
1700                 INIT_LIST_HEAD(&per_cpu(xskmap_flush_list, cpu));
1701         return 0;
1702
1703 out_pernet:
1704         unregister_pernet_subsys(&xsk_net_ops);
1705 out_sk:
1706         sock_unregister(PF_XDP);
1707 out_proto:
1708         proto_unregister(&xsk_proto);
1709 out:
1710         return err;
1711 }
1712
1713 fs_initcall(xsk_init);