net/packet/af_packet.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   4  *              operating system.  INET is implemented using the  BSD Socket
   5  *              interface as the means of communication with the user level.
   6  *
   7  *              PACKET - implements raw packet sockets.
   8  *
   9  * Authors:     Ross Biro
  10  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  11  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  12  *
  13  * Fixes:
  14  *              Alan Cox        :       verify_area() now used correctly
  15  *              Alan Cox        :       new skbuff lists, look ma no backlogs!
  16  *              Alan Cox        :       tidied skbuff lists.
  17  *              Alan Cox        :       Now uses generic datagram routines I
  18  *                                      added. Also fixed the peek/read crash
  19  *                                      from all old Linux datagram code.
  20  *              Alan Cox        :       Uses the improved datagram code.
  21  *              Alan Cox        :       Added NULL's for socket options.
  22  *              Alan Cox        :       Re-commented the code.
  23  *              Alan Cox        :       Use new kernel side addressing
  24  *              Rob Janssen     :       Correct MTU usage.
  25  *              Dave Platt      :       Counter leaks caused by incorrect
  26  *                                      interrupt locking and some slightly
  27  *                                      dubious gcc output. Can you read
  28  *                                      compiler: it said _VOLATILE_
  29  *      Richard Kooijman        :       Timestamp fixes.
  30  *              Alan Cox        :       New buffers. Use sk->mac.raw.
  31  *              Alan Cox        :       sendmsg/recvmsg support.
  32  *              Alan Cox        :       Protocol setting support
  33  *      Alexey Kuznetsov        :       Untied from IPv4 stack.
  34  *      Cyrus Durgin            :       Fixed kerneld for kmod.
  35  *      Michal Ostrowski        :       Module initialization cleanup.
  36  *         Ulises Alonso        :       Frame number limit removal and
  37  *                                      packet_set_ring memory leak.
  38  *              Eric Biederman  :       Allow for > 8 byte hardware addresses.
  39  *                                      The convention is that longer addresses
  40  *                                      will simply extend the hardware address
  41  *                                      byte arrays at the end of sockaddr_ll
  42  *                                      and packet_mreq.
  43  *              Johann Baudy    :       Added TX RING.
  44  *              Chetan Loke     :       Implemented TPACKET_V3 block abstraction
  45  *                                      layer.
  46  *                                      Copyright (C) 2011, <lokec@ccs.neu.edu>
  47  */
  48
  49 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  50
  51 #include <linux/ethtool.h>
  52 #include <linux/filter.h>
  53 #include <linux/types.h>
  54 #include <linux/mm.h>
  55 #include <linux/capability.h>
  56 #include <linux/fcntl.h>
  57 #include <linux/socket.h>
  58 #include <linux/in.h>
  59 #include <linux/inet.h>
  60 #include <linux/netdevice.h>
  61 #include <linux/if_packet.h>
  62 #include <linux/wireless.h>
  63 #include <linux/kernel.h>
  64 #include <linux/kmod.h>
  65 #include <linux/slab.h>
  66 #include <linux/vmalloc.h>
  67 #include <net/net_namespace.h>
  68 #include <net/ip.h>
  69 #include <net/protocol.h>
  70 #include <linux/skbuff.h>
  71 #include <net/sock.h>
  72 #include <linux/errno.h>
  73 #include <linux/timer.h>
  74 #include <linux/uaccess.h>
  75 #include <asm/ioctls.h>
  76 #include <asm/page.h>
  77 #include <asm/cacheflush.h>
  78 #include <asm/io.h>
  79 #include <linux/proc_fs.h>
  80 #include <linux/seq_file.h>
  81 #include <linux/poll.h>
  82 #include <linux/module.h>
  83 #include <linux/init.h>
  84 #include <linux/mutex.h>
  85 #include <linux/if_vlan.h>
  86 #include <linux/virtio_net.h>
  87 #include <linux/errqueue.h>
  88 #include <linux/net_tstamp.h>
  89 #include <linux/percpu.h>
  90 #ifdef CONFIG_INET
  91 #include <net/inet_common.h>
  92 #endif
  93 #include <linux/bpf.h>
  94 #include <net/compat.h>
  95 #include <linux/netfilter_netdev.h>
  96
  97 #include "internal.h"
  98
  99 /*
 100    Assumptions:
 101    - If the device has no dev->header_ops->create, there is no LL header
 102      visible above the device. In this case, its hard_header_len should be 0.
 103      The device may prepend its own header internally. In this case, its
 104      needed_headroom should be set to the space needed for it to add its
 105      internal header.
 106      For example, a WiFi driver pretending to be an Ethernet driver should
 107      set its hard_header_len to be the Ethernet header length, and set its
 108      needed_headroom to be (the real WiFi header length - the fake Ethernet
 109      header length).
 110    - packet socket receives packets with pulled ll header,
 111      so that SOCK_RAW should push it back.
 112
 113 On receive:
 114 -----------
 115
 116 Incoming, dev_has_header(dev) == true
 117    mac_header -> ll header
 118    data       -> data
 119
 120 Outgoing, dev_has_header(dev) == true
 121    mac_header -> ll header
 122    data       -> ll header
 123
 124 Incoming, dev_has_header(dev) == false
 125    mac_header -> data
 126      However drivers often make it point to the ll header.
 127      This is incorrect because the ll header should be invisible to us.
 128    data       -> data
 129
 130 Outgoing, dev_has_header(dev) == false
 131    mac_header -> data. ll header is invisible to us.
 132    data       -> data
 133
 134 Resume
 135   If dev_has_header(dev) == false we are unable to restore the ll header,
 136     because it is invisible to us.
 137
 138
 139 On transmit:
 140 ------------
 141
 142 dev_has_header(dev) == true
 143    mac_header -> ll header
 144    data       -> ll header
 145
 146 dev_has_header(dev) == false (ll header is invisible to us)
 147    mac_header -> data
 148    data       -> data
 149
 150    We should set network_header on output to the correct position,
 151    packet classifier depends on it.
 152  */
 153
 154 /* Private packet socket structures. */
 155
 156 /* identical to struct packet_mreq except it has
 157  * a longer address field.
 158  */
 159 struct packet_mreq_max {
 160         int             mr_ifindex;
 161         unsigned short  mr_type;
 162         unsigned short  mr_alen;
 163         unsigned char   mr_address[MAX_ADDR_LEN];
 164 };
 165
 166 union tpacket_uhdr {
 167         struct tpacket_hdr  *h1;
 168         struct tpacket2_hdr *h2;
 169         struct tpacket3_hdr *h3;
 170         void *raw;
 171 };
 172
 173 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
 174                 int closing, int tx_ring);
 175
 176 #define V3_ALIGNMENT    (8)
 177
 178 #define BLK_HDR_LEN     (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
 179
 180 #define BLK_PLUS_PRIV(sz_of_priv) \
 181         (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
 182
 183 #define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
 184 #define BLOCK_NUM_PKTS(x)       ((x)->hdr.bh1.num_pkts)
 185 #define BLOCK_O2FP(x)           ((x)->hdr.bh1.offset_to_first_pkt)
 186 #define BLOCK_LEN(x)            ((x)->hdr.bh1.blk_len)
 187 #define BLOCK_SNUM(x)           ((x)->hdr.bh1.seq_num)
 188 #define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
 189
 190 struct packet_sock;
 191 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
 192                        struct packet_type *pt, struct net_device *orig_dev);
 193
 194 static void *packet_previous_frame(struct packet_sock *po,
 195                 struct packet_ring_buffer *rb,
 196                 int status);
 197 static void packet_increment_head(struct packet_ring_buffer *buff);
 198 static int prb_curr_blk_in_use(struct tpacket_block_desc *);
 199 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
 200                         struct packet_sock *);
 201 static void prb_retire_current_block(struct tpacket_kbdq_core *,
 202                 struct packet_sock *, unsigned int status);
 203 static int prb_queue_frozen(struct tpacket_kbdq_core *);
 204 static void prb_open_block(struct tpacket_kbdq_core *,
 205                 struct tpacket_block_desc *);
 206 static void prb_retire_rx_blk_timer_expired(struct timer_list *);
 207 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
 208 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
 209 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
 210                 struct tpacket3_hdr *);
 211 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
 212                 struct tpacket3_hdr *);
 213 static void packet_flush_mclist(struct sock *sk);
 214 static u16 packet_pick_tx_queue(struct sk_buff *skb);
 215
 216 struct packet_skb_cb {
 217         union {
 218                 struct sockaddr_pkt pkt;
 219                 union {
 220                         /* Trick: alias skb original length with
 221                          * ll.sll_family and ll.protocol in order
 222                          * to save room.
 223                          */
 224                         unsigned int origlen;
 225                         struct sockaddr_ll ll;
 226                 };
 227         } sa;
 228 };
 229
 230 #define vio_le() virtio_legacy_is_little_endian()
 231
 232 #define PACKET_SKB_CB(__skb)    ((struct packet_skb_cb *)((__skb)->cb))
 233
 234 #define GET_PBDQC_FROM_RB(x)    ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
 235 #define GET_PBLOCK_DESC(x, bid) \
 236         ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
 237 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x)       \
 238         ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
 239 #define GET_NEXT_PRB_BLK_NUM(x) \
 240         (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
 241         ((x)->kactive_blk_num+1) : 0)
 242
 243 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
 244 static void __fanout_link(struct sock *sk, struct packet_sock *po);
 245
 246 #ifdef CONFIG_NETFILTER_EGRESS
 247 static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb)
 248 {
 249         struct sk_buff *next, *head = NULL, *tail;
 250         int rc;
 251
 252         rcu_read_lock();
 253         for (; skb != NULL; skb = next) {
 254                 next = skb->next;
 255                 skb_mark_not_on_list(skb);
 256
 257                 if (!nf_hook_egress(skb, &rc, skb->dev))
 258                         continue;
 259
 260                 if (!head)
 261                         head = skb;
 262                 else
 263                         tail->next = skb;
 264
 265                 tail = skb;
 266         }
 267         rcu_read_unlock();
 268
 269         return head;
 270 }
 271 #endif
 272
 273 static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb)
 274 {
 275         if (!packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS))
 276                 return dev_queue_xmit(skb);
 277
 278 #ifdef CONFIG_NETFILTER_EGRESS
 279         if (nf_hook_egress_active()) {
 280                 skb = nf_hook_direct_egress(skb);
 281                 if (!skb)
 282                         return NET_XMIT_DROP;
 283         }
 284 #endif
 285         return dev_direct_xmit(skb, packet_pick_tx_queue(skb));
 286 }
 287
 288 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
 289 {
 290         struct net_device *dev;
 291
 292         rcu_read_lock();
 293         dev = rcu_dereference(po->cached_dev);
 294         dev_hold(dev);
 295         rcu_read_unlock();
 296
 297         return dev;
 298 }
 299
 300 static void packet_cached_dev_assign(struct packet_sock *po,
 301                                      struct net_device *dev)
 302 {
 303         rcu_assign_pointer(po->cached_dev, dev);
 304 }
 305
 306 static void packet_cached_dev_reset(struct packet_sock *po)
 307 {
 308         RCU_INIT_POINTER(po->cached_dev, NULL);
 309 }
 310
 311 static u16 packet_pick_tx_queue(struct sk_buff *skb)
 312 {
 313         struct net_device *dev = skb->dev;
 314         const struct net_device_ops *ops = dev->netdev_ops;
 315         int cpu = raw_smp_processor_id();
 316         u16 queue_index;
 317
 318 #ifdef CONFIG_XPS
 319         skb->sender_cpu = cpu + 1;
 320 #endif
 321         skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues);
 322         if (ops->ndo_select_queue) {
 323                 queue_index = ops->ndo_select_queue(dev, skb, NULL);
 324                 queue_index = netdev_cap_txqueue(dev, queue_index);
 325         } else {
 326                 queue_index = netdev_pick_tx(dev, skb, NULL);
 327         }
 328
 329         return queue_index;
 330 }
 331
 332 /* __register_prot_hook must be invoked through register_prot_hook
 333  * or from a context in which asynchronous accesses to the packet
 334  * socket is not possible (packet_create()).
 335  */
 336 static void __register_prot_hook(struct sock *sk)
 337 {
 338         struct packet_sock *po = pkt_sk(sk);
 339
 340         if (!packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
 341                 if (po->fanout)
 342                         __fanout_link(sk, po);
 343                 else
 344                         dev_add_pack(&po->prot_hook);
 345
 346                 sock_hold(sk);
 347                 packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 1);
 348         }
 349 }
 350
 351 static void register_prot_hook(struct sock *sk)
 352 {
 353         lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
 354         __register_prot_hook(sk);
 355 }
 356
 357 /* If the sync parameter is true, we will temporarily drop
 358  * the po->bind_lock and do a synchronize_net to make sure no
 359  * asynchronous packet processing paths still refer to the elements
 360  * of po->prot_hook.  If the sync parameter is false, it is the
 361  * callers responsibility to take care of this.
 362  */
 363 static void __unregister_prot_hook(struct sock *sk, bool sync)
 364 {
 365         struct packet_sock *po = pkt_sk(sk);
 366
 367         lockdep_assert_held_once(&po->bind_lock);
 368
 369         packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 0);
 370
 371         if (po->fanout)
 372                 __fanout_unlink(sk, po);
 373         else
 374                 __dev_remove_pack(&po->prot_hook);
 375
 376         __sock_put(sk);
 377
 378         if (sync) {
 379                 spin_unlock(&po->bind_lock);
 380                 synchronize_net();
 381                 spin_lock(&po->bind_lock);
 382         }
 383 }
 384
 385 static void unregister_prot_hook(struct sock *sk, bool sync)
 386 {
 387         struct packet_sock *po = pkt_sk(sk);
 388
 389         if (packet_sock_flag(po, PACKET_SOCK_RUNNING))
 390                 __unregister_prot_hook(sk, sync);
 391 }
 392
 393 static inline struct page * __pure pgv_to_page(void *addr)
 394 {
 395         if (is_vmalloc_addr(addr))
 396                 return vmalloc_to_page(addr);
 397         return virt_to_page(addr);
 398 }
 399
 400 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
 401 {
 402         union tpacket_uhdr h;
 403
 404         h.raw = frame;
 405         switch (po->tp_version) {
 406         case TPACKET_V1:
 407                 h.h1->tp_status = status;
 408                 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
 409                 break;
 410         case TPACKET_V2:
 411                 h.h2->tp_status = status;
 412                 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
 413                 break;
 414         case TPACKET_V3:
 415                 h.h3->tp_status = status;
 416                 flush_dcache_page(pgv_to_page(&h.h3->tp_status));
 417                 break;
 418         default:
 419                 WARN(1, "TPACKET version not supported.\n");
 420                 BUG();
 421         }
 422
 423         smp_wmb();
 424 }
 425
 426 static int __packet_get_status(const struct packet_sock *po, void *frame)
 427 {
 428         union tpacket_uhdr h;
 429
 430         smp_rmb();
 431
 432         h.raw = frame;
 433         switch (po->tp_version) {
 434         case TPACKET_V1:
 435                 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
 436                 return h.h1->tp_status;
 437         case TPACKET_V2:
 438                 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
 439                 return h.h2->tp_status;
 440         case TPACKET_V3:
 441                 flush_dcache_page(pgv_to_page(&h.h3->tp_status));
 442                 return h.h3->tp_status;
 443         default:
 444                 WARN(1, "TPACKET version not supported.\n");
 445                 BUG();
 446                 return 0;
 447         }
 448 }
 449
 450 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec64 *ts,
 451                                    unsigned int flags)
 452 {
 453         struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
 454
 455         if (shhwtstamps &&
 456             (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
 457             ktime_to_timespec64_cond(shhwtstamps->hwtstamp, ts))
 458                 return TP_STATUS_TS_RAW_HARDWARE;
 459
 460         if ((flags & SOF_TIMESTAMPING_SOFTWARE) &&
 461             ktime_to_timespec64_cond(skb_tstamp(skb), ts))
 462                 return TP_STATUS_TS_SOFTWARE;
 463
 464         return 0;
 465 }
 466
 467 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
 468                                     struct sk_buff *skb)
 469 {
 470         union tpacket_uhdr h;
 471         struct timespec64 ts;
 472         __u32 ts_status;
 473
 474         if (!(ts_status = tpacket_get_timestamp(skb, &ts, READ_ONCE(po->tp_tstamp))))
 475                 return 0;
 476
 477         h.raw = frame;
 478         /*
 479          * versions 1 through 3 overflow the timestamps in y2106, since they
 480          * all store the seconds in a 32-bit unsigned integer.
 481          * If we create a version 4, that should have a 64-bit timestamp,
 482          * either 64-bit seconds + 32-bit nanoseconds, or just 64-bit
 483          * nanoseconds.
 484          */
 485         switch (po->tp_version) {
 486         case TPACKET_V1:
 487                 h.h1->tp_sec = ts.tv_sec;
 488                 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
 489                 break;
 490         case TPACKET_V2:
 491                 h.h2->tp_sec = ts.tv_sec;
 492                 h.h2->tp_nsec = ts.tv_nsec;
 493                 break;
 494         case TPACKET_V3:
 495                 h.h3->tp_sec = ts.tv_sec;
 496                 h.h3->tp_nsec = ts.tv_nsec;
 497                 break;
 498         default:
 499                 WARN(1, "TPACKET version not supported.\n");
 500                 BUG();
 501         }
 502
 503         /* one flush is safe, as both fields always lie on the same cacheline */
 504         flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
 505         smp_wmb();
 506
 507         return ts_status;
 508 }
 509
 510 static void *packet_lookup_frame(const struct packet_sock *po,
 511                                  const struct packet_ring_buffer *rb,
 512                                  unsigned int position,
 513                                  int status)
 514 {
 515         unsigned int pg_vec_pos, frame_offset;
 516         union tpacket_uhdr h;
 517
 518         pg_vec_pos = position / rb->frames_per_block;
 519         frame_offset = position % rb->frames_per_block;
 520
 521         h.raw = rb->pg_vec[pg_vec_pos].buffer +
 522                 (frame_offset * rb->frame_size);
 523
 524         if (status != __packet_get_status(po, h.raw))
 525                 return NULL;
 526
 527         return h.raw;
 528 }
 529
 530 static void *packet_current_frame(struct packet_sock *po,
 531                 struct packet_ring_buffer *rb,
 532                 int status)
 533 {
 534         return packet_lookup_frame(po, rb, rb->head, status);
 535 }
 536
 537 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
 538 {
 539         del_timer_sync(&pkc->retire_blk_timer);
 540 }
 541
 542 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
 543                 struct sk_buff_head *rb_queue)
 544 {
 545         struct tpacket_kbdq_core *pkc;
 546
 547         pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
 548
 549         spin_lock_bh(&rb_queue->lock);
 550         pkc->delete_blk_timer = 1;
 551         spin_unlock_bh(&rb_queue->lock);
 552
 553         prb_del_retire_blk_timer(pkc);
 554 }
 555
 556 static void prb_setup_retire_blk_timer(struct packet_sock *po)
 557 {
 558         struct tpacket_kbdq_core *pkc;
 559
 560         pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
 561         timer_setup(&pkc->retire_blk_timer, prb_retire_rx_blk_timer_expired,
 562                     0);
 563         pkc->retire_blk_timer.expires = jiffies;
 564 }
 565
 566 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
 567                                 int blk_size_in_bytes)
 568 {
 569         struct net_device *dev;
 570         unsigned int mbits, div;
 571         struct ethtool_link_ksettings ecmd;
 572         int err;
 573
 574         rtnl_lock();
 575         dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
 576         if (unlikely(!dev)) {
 577                 rtnl_unlock();
 578                 return DEFAULT_PRB_RETIRE_TOV;
 579         }
 580         err = __ethtool_get_link_ksettings(dev, &ecmd);
 581         rtnl_unlock();
 582         if (err)
 583                 return DEFAULT_PRB_RETIRE_TOV;
 584
 585         /* If the link speed is so slow you don't really
 586          * need to worry about perf anyways
 587          */
 588         if (ecmd.base.speed < SPEED_1000 ||
 589             ecmd.base.speed == SPEED_UNKNOWN)
 590                 return DEFAULT_PRB_RETIRE_TOV;
 591
 592         div = ecmd.base.speed / 1000;
 593         mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
 594
 595         if (div)
 596                 mbits /= div;
 597
 598         if (div)
 599                 return mbits + 1;
 600         return mbits;
 601 }
 602
 603 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
 604                         union tpacket_req_u *req_u)
 605 {
 606         p1->feature_req_word = req_u->req3.tp_feature_req_word;
 607 }
 608
 609 static void init_prb_bdqc(struct packet_sock *po,
 610                         struct packet_ring_buffer *rb,
 611                         struct pgv *pg_vec,
 612                         union tpacket_req_u *req_u)
 613 {
 614         struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
 615         struct tpacket_block_desc *pbd;
 616
 617         memset(p1, 0x0, sizeof(*p1));
 618
 619         p1->knxt_seq_num = 1;
 620         p1->pkbdq = pg_vec;
 621         pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
 622         p1->pkblk_start = pg_vec[0].buffer;
 623         p1->kblk_size = req_u->req3.tp_block_size;
 624         p1->knum_blocks = req_u->req3.tp_block_nr;
 625         p1->hdrlen = po->tp_hdrlen;
 626         p1->version = po->tp_version;
 627         p1->last_kactive_blk_num = 0;
 628         po->stats.stats3.tp_freeze_q_cnt = 0;
 629         if (req_u->req3.tp_retire_blk_tov)
 630                 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
 631         else
 632                 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
 633                                                 req_u->req3.tp_block_size);
 634         p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
 635         p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
 636         rwlock_init(&p1->blk_fill_in_prog_lock);
 637
 638         p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
 639         prb_init_ft_ops(p1, req_u);
 640         prb_setup_retire_blk_timer(po);
 641         prb_open_block(p1, pbd);
 642 }
 643
 644 /*  Do NOT update the last_blk_num first.
 645  *  Assumes sk_buff_head lock is held.
 646  */
 647 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
 648 {
 649         mod_timer(&pkc->retire_blk_timer,
 650                         jiffies + pkc->tov_in_jiffies);
 651         pkc->last_kactive_blk_num = pkc->kactive_blk_num;
 652 }
 653
 654 /*
 655  * Timer logic:
 656  * 1) We refresh the timer only when we open a block.
 657  *    By doing this we don't waste cycles refreshing the timer
 658  *        on packet-by-packet basis.
 659  *
 660  * With a 1MB block-size, on a 1Gbps line, it will take
 661  * i) ~8 ms to fill a block + ii) memcpy etc.
 662  * In this cut we are not accounting for the memcpy time.
 663  *
 664  * So, if the user sets the 'tmo' to 10ms then the timer
 665  * will never fire while the block is still getting filled
 666  * (which is what we want). However, the user could choose
 667  * to close a block early and that's fine.
 668  *
 669  * But when the timer does fire, we check whether or not to refresh it.
 670  * Since the tmo granularity is in msecs, it is not too expensive
 671  * to refresh the timer, lets say every '8' msecs.
 672  * Either the user can set the 'tmo' or we can derive it based on
 673  * a) line-speed and b) block-size.
 674  * prb_calc_retire_blk_tmo() calculates the tmo.
 675  *
 676  */
 677 static void prb_retire_rx_blk_timer_expired(struct timer_list *t)
 678 {
 679         struct packet_sock *po =
 680                 from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer);
 681         struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
 682         unsigned int frozen;
 683         struct tpacket_block_desc *pbd;
 684
 685         spin_lock(&po->sk.sk_receive_queue.lock);
 686
 687         frozen = prb_queue_frozen(pkc);
 688         pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
 689
 690         if (unlikely(pkc->delete_blk_timer))
 691                 goto out;
 692
 693         /* We only need to plug the race when the block is partially filled.
 694          * tpacket_rcv:
 695          *              lock(); increment BLOCK_NUM_PKTS; unlock()
 696          *              copy_bits() is in progress ...
 697          *              timer fires on other cpu:
 698          *              we can't retire the current block because copy_bits
 699          *              is in progress.
 700          *
 701          */
 702         if (BLOCK_NUM_PKTS(pbd)) {
 703                 /* Waiting for skb_copy_bits to finish... */
 704                 write_lock(&pkc->blk_fill_in_prog_lock);
 705                 write_unlock(&pkc->blk_fill_in_prog_lock);
 706         }
 707
 708         if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
 709                 if (!frozen) {
 710                         if (!BLOCK_NUM_PKTS(pbd)) {
 711                                 /* An empty block. Just refresh the timer. */
 712                                 goto refresh_timer;
 713                         }
 714                         prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
 715                         if (!prb_dispatch_next_block(pkc, po))
 716                                 goto refresh_timer;
 717                         else
 718                                 goto out;
 719                 } else {
 720                         /* Case 1. Queue was frozen because user-space was
 721                          *         lagging behind.
 722                          */
 723                         if (prb_curr_blk_in_use(pbd)) {
 724                                 /*
 725                                  * Ok, user-space is still behind.
 726                                  * So just refresh the timer.
 727                                  */
 728                                 goto refresh_timer;
 729                         } else {
 730                                /* Case 2. queue was frozen,user-space caught up,
 731                                 * now the link went idle && the timer fired.
 732                                 * We don't have a block to close.So we open this
 733                                 * block and restart the timer.
 734                                 * opening a block thaws the queue,restarts timer
 735                                 * Thawing/timer-refresh is a side effect.
 736                                 */
 737                                 prb_open_block(pkc, pbd);
 738                                 goto out;
 739                         }
 740                 }
 741         }
 742
 743 refresh_timer:
 744         _prb_refresh_rx_retire_blk_timer(pkc);
 745
 746 out:
 747         spin_unlock(&po->sk.sk_receive_queue.lock);
 748 }
 749
 750 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
 751                 struct tpacket_block_desc *pbd1, __u32 status)
 752 {
 753         /* Flush everything minus the block header */
 754
 755 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
 756         u8 *start, *end;
 757
 758         start = (u8 *)pbd1;
 759
 760         /* Skip the block header(we know header WILL fit in 4K) */
 761         start += PAGE_SIZE;
 762
 763         end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
 764         for (; start < end; start += PAGE_SIZE)
 765                 flush_dcache_page(pgv_to_page(start));
 766
 767         smp_wmb();
 768 #endif
 769
 770         /* Now update the block status. */
 771
 772         BLOCK_STATUS(pbd1) = status;
 773
 774         /* Flush the block header */
 775
 776 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
 777         start = (u8 *)pbd1;
 778         flush_dcache_page(pgv_to_page(start));
 779
 780         smp_wmb();
 781 #endif
 782 }
 783
 784 /*
 785  * Side effect:
 786  *
 787  * 1) flush the block
 788  * 2) Increment active_blk_num
 789  *
 790  * Note:We DONT refresh the timer on purpose.
 791  *      Because almost always the next block will be opened.
 792  */
 793 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
 794                 struct tpacket_block_desc *pbd1,
 795                 struct packet_sock *po, unsigned int stat)
 796 {
 797         __u32 status = TP_STATUS_USER | stat;
 798
 799         struct tpacket3_hdr *last_pkt;
 800         struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
 801         struct sock *sk = &po->sk;
 802
 803         if (atomic_read(&po->tp_drops))
 804                 status |= TP_STATUS_LOSING;
 805
 806         last_pkt = (struct tpacket3_hdr *)pkc1->prev;
 807         last_pkt->tp_next_offset = 0;
 808
 809         /* Get the ts of the last pkt */
 810         if (BLOCK_NUM_PKTS(pbd1)) {
 811                 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
 812                 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
 813         } else {
 814                 /* Ok, we tmo'd - so get the current time.
 815                  *
 816                  * It shouldn't really happen as we don't close empty
 817                  * blocks. See prb_retire_rx_blk_timer_expired().
 818                  */
 819                 struct timespec64 ts;
 820                 ktime_get_real_ts64(&ts);
 821                 h1->ts_last_pkt.ts_sec = ts.tv_sec;
 822                 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
 823         }
 824
 825         smp_wmb();
 826
 827         /* Flush the block */
 828         prb_flush_block(pkc1, pbd1, status);
 829
 830         sk->sk_data_ready(sk);
 831
 832         pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
 833 }
 834
 835 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
 836 {
 837         pkc->reset_pending_on_curr_blk = 0;
 838 }
 839
 840 /*
 841  * Side effect of opening a block:
 842  *
 843  * 1) prb_queue is thawed.
 844  * 2) retire_blk_timer is refreshed.
 845  *
 846  */
 847 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
 848         struct tpacket_block_desc *pbd1)
 849 {
 850         struct timespec64 ts;
 851         struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
 852
 853         smp_rmb();
 854
 855         /* We could have just memset this but we will lose the
 856          * flexibility of making the priv area sticky
 857          */
 858
 859         BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
 860         BLOCK_NUM_PKTS(pbd1) = 0;
 861         BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
 862
 863         ktime_get_real_ts64(&ts);
 864
 865         h1->ts_first_pkt.ts_sec = ts.tv_sec;
 866         h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
 867
 868         pkc1->pkblk_start = (char *)pbd1;
 869         pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
 870
 871         BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
 872         BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
 873
 874         pbd1->version = pkc1->version;
 875         pkc1->prev = pkc1->nxt_offset;
 876         pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
 877
 878         prb_thaw_queue(pkc1);
 879         _prb_refresh_rx_retire_blk_timer(pkc1);
 880
 881         smp_wmb();
 882 }
 883
 884 /*
 885  * Queue freeze logic:
 886  * 1) Assume tp_block_nr = 8 blocks.
 887  * 2) At time 't0', user opens Rx ring.
 888  * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
 889  * 4) user-space is either sleeping or processing block '0'.
 890  * 5) tpacket_rcv is currently filling block '7', since there is no space left,
 891  *    it will close block-7,loop around and try to fill block '0'.
 892  *    call-flow:
 893  *    __packet_lookup_frame_in_block
 894  *      prb_retire_current_block()
 895  *      prb_dispatch_next_block()
 896  *        |->(BLOCK_STATUS == USER) evaluates to true
 897  *    5.1) Since block-0 is currently in-use, we just freeze the queue.
 898  * 6) Now there are two cases:
 899  *    6.1) Link goes idle right after the queue is frozen.
 900  *         But remember, the last open_block() refreshed the timer.
 901  *         When this timer expires,it will refresh itself so that we can
 902  *         re-open block-0 in near future.
 903  *    6.2) Link is busy and keeps on receiving packets. This is a simple
 904  *         case and __packet_lookup_frame_in_block will check if block-0
 905  *         is free and can now be re-used.
 906  */
 907 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
 908                                   struct packet_sock *po)
 909 {
 910         pkc->reset_pending_on_curr_blk = 1;
 911         po->stats.stats3.tp_freeze_q_cnt++;
 912 }
 913
 914 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
 915
 916 /*
 917  * If the next block is free then we will dispatch it
 918  * and return a good offset.
 919  * Else, we will freeze the queue.
 920  * So, caller must check the return value.
 921  */
 922 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
 923                 struct packet_sock *po)
 924 {
 925         struct tpacket_block_desc *pbd;
 926
 927         smp_rmb();
 928
 929         /* 1. Get current block num */
 930         pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
 931
 932         /* 2. If this block is currently in_use then freeze the queue */
 933         if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
 934                 prb_freeze_queue(pkc, po);
 935                 return NULL;
 936         }
 937
 938         /*
 939          * 3.
 940          * open this block and return the offset where the first packet
 941          * needs to get stored.
 942          */
 943         prb_open_block(pkc, pbd);
 944         return (void *)pkc->nxt_offset;
 945 }
 946
 947 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
 948                 struct packet_sock *po, unsigned int status)
 949 {
 950         struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
 951
 952         /* retire/close the current block */
 953         if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
 954                 /*
 955                  * Plug the case where copy_bits() is in progress on
 956                  * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
 957                  * have space to copy the pkt in the current block and
 958                  * called prb_retire_current_block()
 959                  *
 960                  * We don't need to worry about the TMO case because
 961                  * the timer-handler already handled this case.
 962                  */
 963                 if (!(status & TP_STATUS_BLK_TMO)) {
 964                         /* Waiting for skb_copy_bits to finish... */
 965                         write_lock(&pkc->blk_fill_in_prog_lock);
 966                         write_unlock(&pkc->blk_fill_in_prog_lock);
 967                 }
 968                 prb_close_block(pkc, pbd, po, status);
 969                 return;
 970         }
 971 }
 972
 973 static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
 974 {
 975         return TP_STATUS_USER & BLOCK_STATUS(pbd);
 976 }
 977
 978 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
 979 {
 980         return pkc->reset_pending_on_curr_blk;
 981 }
 982
 983 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
 984         __releases(&pkc->blk_fill_in_prog_lock)
 985 {
 986         struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
 987
 988         read_unlock(&pkc->blk_fill_in_prog_lock);
 989 }
 990
 991 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
 992                         struct tpacket3_hdr *ppd)
 993 {
 994         ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
 995 }
 996
 997 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
 998                         struct tpacket3_hdr *ppd)
 999 {
1000         ppd->hv1.tp_rxhash = 0;
1001 }
1002
1003 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
1004                         struct tpacket3_hdr *ppd)
1005 {
1006         if (skb_vlan_tag_present(pkc->skb)) {
1007                 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
1008                 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1009                 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1010         } else {
1011                 ppd->hv1.tp_vlan_tci = 0;
1012                 ppd->hv1.tp_vlan_tpid = 0;
1013                 ppd->tp_status = TP_STATUS_AVAILABLE;
1014         }
1015 }
1016
1017 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
1018                         struct tpacket3_hdr *ppd)
1019 {
1020         ppd->hv1.tp_padding = 0;
1021         prb_fill_vlan_info(pkc, ppd);
1022
1023         if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1024                 prb_fill_rxhash(pkc, ppd);
1025         else
1026                 prb_clear_rxhash(pkc, ppd);
1027 }
1028
1029 static void prb_fill_curr_block(char *curr,
1030                                 struct tpacket_kbdq_core *pkc,
1031                                 struct tpacket_block_desc *pbd,
1032                                 unsigned int len)
1033         __acquires(&pkc->blk_fill_in_prog_lock)
1034 {
1035         struct tpacket3_hdr *ppd;
1036
1037         ppd  = (struct tpacket3_hdr *)curr;
1038         ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1039         pkc->prev = curr;
1040         pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1041         BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1042         BLOCK_NUM_PKTS(pbd) += 1;
1043         read_lock(&pkc->blk_fill_in_prog_lock);
1044         prb_run_all_ft_ops(pkc, ppd);
1045 }
1046
1047 /* Assumes caller has the sk->rx_queue.lock */
1048 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1049                                             struct sk_buff *skb,
1050                                             unsigned int len
1051                                             )
1052 {
1053         struct tpacket_kbdq_core *pkc;
1054         struct tpacket_block_desc *pbd;
1055         char *curr, *end;
1056
1057         pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1058         pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1059
1060         /* Queue is frozen when user space is lagging behind */
1061         if (prb_queue_frozen(pkc)) {
1062                 /*
1063                  * Check if that last block which caused the queue to freeze,
1064                  * is still in_use by user-space.
1065                  */
1066                 if (prb_curr_blk_in_use(pbd)) {
1067                         /* Can't record this packet */
1068                         return NULL;
1069                 } else {
1070                         /*
1071                          * Ok, the block was released by user-space.
1072                          * Now let's open that block.
1073                          * opening a block also thaws the queue.
1074                          * Thawing is a side effect.
1075                          */
1076                         prb_open_block(pkc, pbd);
1077                 }
1078         }
1079
1080         smp_mb();
1081         curr = pkc->nxt_offset;
1082         pkc->skb = skb;
1083         end = (char *)pbd + pkc->kblk_size;
1084
1085         /* first try the current block */
1086         if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1087                 prb_fill_curr_block(curr, pkc, pbd, len);
1088                 return (void *)curr;
1089         }
1090
1091         /* Ok, close the current block */
1092         prb_retire_current_block(pkc, po, 0);
1093
1094         /* Now, try to dispatch the next block */
1095         curr = (char *)prb_dispatch_next_block(pkc, po);
1096         if (curr) {
1097                 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1098                 prb_fill_curr_block(curr, pkc, pbd, len);
1099                 return (void *)curr;
1100         }
1101
1102         /*
1103          * No free blocks are available.user_space hasn't caught up yet.
1104          * Queue was just frozen and now this packet will get dropped.
1105          */
1106         return NULL;
1107 }
1108
1109 static void *packet_current_rx_frame(struct packet_sock *po,
1110                                             struct sk_buff *skb,
1111                                             int status, unsigned int len)
1112 {
1113         char *curr = NULL;
1114         switch (po->tp_version) {
1115         case TPACKET_V1:
1116         case TPACKET_V2:
1117                 curr = packet_lookup_frame(po, &po->rx_ring,
1118                                         po->rx_ring.head, status);
1119                 return curr;
1120         case TPACKET_V3:
1121                 return __packet_lookup_frame_in_block(po, skb, len);
1122         default:
1123                 WARN(1, "TPACKET version not supported\n");
1124                 BUG();
1125                 return NULL;
1126         }
1127 }
1128
1129 static void *prb_lookup_block(const struct packet_sock *po,
1130                               const struct packet_ring_buffer *rb,
1131                               unsigned int idx,
1132                               int status)
1133 {
1134         struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
1135         struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1136
1137         if (status != BLOCK_STATUS(pbd))
1138                 return NULL;
1139         return pbd;
1140 }
1141
1142 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1143 {
1144         unsigned int prev;
1145         if (rb->prb_bdqc.kactive_blk_num)
1146                 prev = rb->prb_bdqc.kactive_blk_num-1;
1147         else
1148                 prev = rb->prb_bdqc.knum_blocks-1;
1149         return prev;
1150 }
1151
1152 /* Assumes caller has held the rx_queue.lock */
1153 static void *__prb_previous_block(struct packet_sock *po,
1154                                          struct packet_ring_buffer *rb,
1155                                          int status)
1156 {
1157         unsigned int previous = prb_previous_blk_num(rb);
1158         return prb_lookup_block(po, rb, previous, status);
1159 }
1160
1161 static void *packet_previous_rx_frame(struct packet_sock *po,
1162                                              struct packet_ring_buffer *rb,
1163                                              int status)
1164 {
1165         if (po->tp_version <= TPACKET_V2)
1166                 return packet_previous_frame(po, rb, status);
1167
1168         return __prb_previous_block(po, rb, status);
1169 }
1170
1171 static void packet_increment_rx_head(struct packet_sock *po,
1172                                             struct packet_ring_buffer *rb)
1173 {
1174         switch (po->tp_version) {
1175         case TPACKET_V1:
1176         case TPACKET_V2:
1177                 return packet_increment_head(rb);
1178         case TPACKET_V3:
1179         default:
1180                 WARN(1, "TPACKET version not supported.\n");
1181                 BUG();
1182                 return;
1183         }
1184 }
1185
1186 static void *packet_previous_frame(struct packet_sock *po,
1187                 struct packet_ring_buffer *rb,
1188                 int status)
1189 {
1190         unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1191         return packet_lookup_frame(po, rb, previous, status);
1192 }
1193
1194 static void packet_increment_head(struct packet_ring_buffer *buff)
1195 {
1196         buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1197 }
1198
1199 static void packet_inc_pending(struct packet_ring_buffer *rb)
1200 {
1201         this_cpu_inc(*rb->pending_refcnt);
1202 }
1203
1204 static void packet_dec_pending(struct packet_ring_buffer *rb)
1205 {
1206         this_cpu_dec(*rb->pending_refcnt);
1207 }
1208
1209 static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1210 {
1211         unsigned int refcnt = 0;
1212         int cpu;
1213
1214         /* We don't use pending refcount in rx_ring. */
1215         if (rb->pending_refcnt == NULL)
1216                 return 0;
1217
1218         for_each_possible_cpu(cpu)
1219                 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1220
1221         return refcnt;
1222 }
1223
1224 static int packet_alloc_pending(struct packet_sock *po)
1225 {
1226         po->rx_ring.pending_refcnt = NULL;
1227
1228         po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1229         if (unlikely(po->tx_ring.pending_refcnt == NULL))
1230                 return -ENOBUFS;
1231
1232         return 0;
1233 }
1234
1235 static void packet_free_pending(struct packet_sock *po)
1236 {
1237         free_percpu(po->tx_ring.pending_refcnt);
1238 }
1239
1240 #define ROOM_POW_OFF    2
1241 #define ROOM_NONE       0x0
1242 #define ROOM_LOW        0x1
1243 #define ROOM_NORMAL     0x2
1244
1245 static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
1246 {
1247         int idx, len;
1248
1249         len = READ_ONCE(po->rx_ring.frame_max) + 1;
1250         idx = READ_ONCE(po->rx_ring.head);
1251         if (pow_off)
1252                 idx += len >> pow_off;
1253         if (idx >= len)
1254                 idx -= len;
1255         return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1256 }
1257
1258 static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
1259 {
1260         int idx, len;
1261
1262         len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
1263         idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
1264         if (pow_off)
1265                 idx += len >> pow_off;
1266         if (idx >= len)
1267                 idx -= len;
1268         return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1269 }
1270
1271 static int __packet_rcv_has_room(const struct packet_sock *po,
1272                                  const struct sk_buff *skb)
1273 {
1274         const struct sock *sk = &po->sk;
1275         int ret = ROOM_NONE;
1276
1277         if (po->prot_hook.func != tpacket_rcv) {
1278                 int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1279                 int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc)
1280                                    - (skb ? skb->truesize : 0);
1281
1282                 if (avail > (rcvbuf >> ROOM_POW_OFF))
1283                         return ROOM_NORMAL;
1284                 else if (avail > 0)
1285                         return ROOM_LOW;
1286                 else
1287                         return ROOM_NONE;
1288         }
1289
1290         if (po->tp_version == TPACKET_V3) {
1291                 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1292                         ret = ROOM_NORMAL;
1293                 else if (__tpacket_v3_has_room(po, 0))
1294                         ret = ROOM_LOW;
1295         } else {
1296                 if (__tpacket_has_room(po, ROOM_POW_OFF))
1297                         ret = ROOM_NORMAL;
1298                 else if (__tpacket_has_room(po, 0))
1299                         ret = ROOM_LOW;
1300         }
1301
1302         return ret;
1303 }
1304
1305 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1306 {
1307         bool pressure;
1308         int ret;
1309
1310         ret = __packet_rcv_has_room(po, skb);
1311         pressure = ret != ROOM_NORMAL;
1312
1313         if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) != pressure)
1314                 packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, pressure);
1315
1316         return ret;
1317 }
1318
1319 static void packet_rcv_try_clear_pressure(struct packet_sock *po)
1320 {
1321         if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) &&
1322             __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
1323                 packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, false);
1324 }
1325
1326 static void packet_sock_destruct(struct sock *sk)
1327 {
1328         skb_queue_purge(&sk->sk_error_queue);
1329
1330         WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1331         WARN_ON(refcount_read(&sk->sk_wmem_alloc));
1332
1333         if (!sock_flag(sk, SOCK_DEAD)) {
1334                 pr_err("Attempt to release alive packet socket: %p\n", sk);
1335                 return;
1336         }
1337 }
1338
1339 static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1340 {
1341         u32 *history = po->rollover->history;
1342         u32 victim, rxhash;
1343         int i, count = 0;
1344
1345         rxhash = skb_get_hash(skb);
1346         for (i = 0; i < ROLLOVER_HLEN; i++)
1347                 if (READ_ONCE(history[i]) == rxhash)
1348                         count++;
1349
1350         victim = get_random_u32_below(ROLLOVER_HLEN);
1351
1352         /* Avoid dirtying the cache line if possible */
1353         if (READ_ONCE(history[victim]) != rxhash)
1354                 WRITE_ONCE(history[victim], rxhash);
1355
1356         return count > (ROLLOVER_HLEN >> 1);
1357 }
1358
1359 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1360                                       struct sk_buff *skb,
1361                                       unsigned int num)
1362 {
1363         return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
1364 }
1365
1366 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1367                                     struct sk_buff *skb,
1368                                     unsigned int num)
1369 {
1370         unsigned int val = atomic_inc_return(&f->rr_cur);
1371
1372         return val % num;
1373 }
1374
1375 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1376                                      struct sk_buff *skb,
1377                                      unsigned int num)
1378 {
1379         return smp_processor_id() % num;
1380 }
1381
1382 static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1383                                      struct sk_buff *skb,
1384                                      unsigned int num)
1385 {
1386         return get_random_u32_below(num);
1387 }
1388
1389 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1390                                           struct sk_buff *skb,
1391                                           unsigned int idx, bool try_self,
1392                                           unsigned int num)
1393 {
1394         struct packet_sock *po, *po_next, *po_skip = NULL;
1395         unsigned int i, j, room = ROOM_NONE;
1396
1397         po = pkt_sk(rcu_dereference(f->arr[idx]));
1398
1399         if (try_self) {
1400                 room = packet_rcv_has_room(po, skb);
1401                 if (room == ROOM_NORMAL ||
1402                     (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1403                         return idx;
1404                 po_skip = po;
1405         }
1406
1407         i = j = min_t(int, po->rollover->sock, num - 1);
1408         do {
1409                 po_next = pkt_sk(rcu_dereference(f->arr[i]));
1410                 if (po_next != po_skip &&
1411                     !packet_sock_flag(po_next, PACKET_SOCK_PRESSURE) &&
1412                     packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
1413                         if (i != j)
1414                                 po->rollover->sock = i;
1415                         atomic_long_inc(&po->rollover->num);
1416                         if (room == ROOM_LOW)
1417                                 atomic_long_inc(&po->rollover->num_huge);
1418                         return i;
1419                 }
1420
1421                 if (++i == num)
1422                         i = 0;
1423         } while (i != j);
1424
1425         atomic_long_inc(&po->rollover->num_failed);
1426         return idx;
1427 }
1428
1429 static unsigned int fanout_demux_qm(struct packet_fanout *f,
1430                                     struct sk_buff *skb,
1431                                     unsigned int num)
1432 {
1433         return skb_get_queue_mapping(skb) % num;
1434 }
1435
1436 static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1437                                      struct sk_buff *skb,
1438                                      unsigned int num)
1439 {
1440         struct bpf_prog *prog;
1441         unsigned int ret = 0;
1442
1443         rcu_read_lock();
1444         prog = rcu_dereference(f->bpf_prog);
1445         if (prog)
1446                 ret = bpf_prog_run_clear_cb(prog, skb) % num;
1447         rcu_read_unlock();
1448
1449         return ret;
1450 }
1451
1452 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1453 {
1454         return f->flags & (flag >> 8);
1455 }
1456
1457 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1458                              struct packet_type *pt, struct net_device *orig_dev)
1459 {
1460         struct packet_fanout *f = pt->af_packet_priv;
1461         unsigned int num = READ_ONCE(f->num_members);
1462         struct net *net = read_pnet(&f->net);
1463         struct packet_sock *po;
1464         unsigned int idx;
1465
1466         if (!net_eq(dev_net(dev), net) || !num) {
1467                 kfree_skb(skb);
1468                 return 0;
1469         }
1470
1471         if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1472                 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
1473                 if (!skb)
1474                         return 0;
1475         }
1476         switch (f->type) {
1477         case PACKET_FANOUT_HASH:
1478         default:
1479                 idx = fanout_demux_hash(f, skb, num);
1480                 break;
1481         case PACKET_FANOUT_LB:
1482                 idx = fanout_demux_lb(f, skb, num);
1483                 break;
1484         case PACKET_FANOUT_CPU:
1485                 idx = fanout_demux_cpu(f, skb, num);
1486                 break;
1487         case PACKET_FANOUT_RND:
1488                 idx = fanout_demux_rnd(f, skb, num);
1489                 break;
1490         case PACKET_FANOUT_QM:
1491                 idx = fanout_demux_qm(f, skb, num);
1492                 break;
1493         case PACKET_FANOUT_ROLLOVER:
1494                 idx = fanout_demux_rollover(f, skb, 0, false, num);
1495                 break;
1496         case PACKET_FANOUT_CBPF:
1497         case PACKET_FANOUT_EBPF:
1498                 idx = fanout_demux_bpf(f, skb, num);
1499                 break;
1500         }
1501
1502         if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1503                 idx = fanout_demux_rollover(f, skb, idx, true, num);
1504
1505         po = pkt_sk(rcu_dereference(f->arr[idx]));
1506         return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1507 }
1508
1509 DEFINE_MUTEX(fanout_mutex);
1510 EXPORT_SYMBOL_GPL(fanout_mutex);
1511 static LIST_HEAD(fanout_list);
1512 static u16 fanout_next_id;
1513
1514 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1515 {
1516         struct packet_fanout *f = po->fanout;
1517
1518         spin_lock(&f->lock);
1519         rcu_assign_pointer(f->arr[f->num_members], sk);
1520         smp_wmb();
1521         f->num_members++;
1522         if (f->num_members == 1)
1523                 dev_add_pack(&f->prot_hook);
1524         spin_unlock(&f->lock);
1525 }
1526
1527 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1528 {
1529         struct packet_fanout *f = po->fanout;
1530         int i;
1531
1532         spin_lock(&f->lock);
1533         for (i = 0; i < f->num_members; i++) {
1534                 if (rcu_dereference_protected(f->arr[i],
1535                                               lockdep_is_held(&f->lock)) == sk)
1536                         break;
1537         }
1538         BUG_ON(i >= f->num_members);
1539         rcu_assign_pointer(f->arr[i],
1540                            rcu_dereference_protected(f->arr[f->num_members - 1],
1541                                                      lockdep_is_held(&f->lock)));
1542         f->num_members--;
1543         if (f->num_members == 0)
1544                 __dev_remove_pack(&f->prot_hook);
1545         spin_unlock(&f->lock);
1546 }
1547
1548 static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1549 {
1550         if (sk->sk_family != PF_PACKET)
1551                 return false;
1552
1553         return ptype->af_packet_priv == pkt_sk(sk)->fanout;
1554 }
1555
1556 static void fanout_init_data(struct packet_fanout *f)
1557 {
1558         switch (f->type) {
1559         case PACKET_FANOUT_LB:
1560                 atomic_set(&f->rr_cur, 0);
1561                 break;
1562         case PACKET_FANOUT_CBPF:
1563         case PACKET_FANOUT_EBPF:
1564                 RCU_INIT_POINTER(f->bpf_prog, NULL);
1565                 break;
1566         }
1567 }
1568
1569 static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1570 {
1571         struct bpf_prog *old;
1572
1573         spin_lock(&f->lock);
1574         old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1575         rcu_assign_pointer(f->bpf_prog, new);
1576         spin_unlock(&f->lock);
1577
1578         if (old) {
1579                 synchronize_net();
1580                 bpf_prog_destroy(old);
1581         }
1582 }
1583
1584 static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data,
1585                                 unsigned int len)
1586 {
1587         struct bpf_prog *new;
1588         struct sock_fprog fprog;
1589         int ret;
1590
1591         if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1592                 return -EPERM;
1593
1594         ret = copy_bpf_fprog_from_user(&fprog, data, len);
1595         if (ret)
1596                 return ret;
1597
1598         ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
1599         if (ret)
1600                 return ret;
1601
1602         __fanout_set_data_bpf(po->fanout, new);
1603         return 0;
1604 }
1605
1606 static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data,
1607                                 unsigned int len)
1608 {
1609         struct bpf_prog *new;
1610         u32 fd;
1611
1612         if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1613                 return -EPERM;
1614         if (len != sizeof(fd))
1615                 return -EINVAL;
1616         if (copy_from_sockptr(&fd, data, len))
1617                 return -EFAULT;
1618
1619         new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
1620         if (IS_ERR(new))
1621                 return PTR_ERR(new);
1622
1623         __fanout_set_data_bpf(po->fanout, new);
1624         return 0;
1625 }
1626
1627 static int fanout_set_data(struct packet_sock *po, sockptr_t data,
1628                            unsigned int len)
1629 {
1630         switch (po->fanout->type) {
1631         case PACKET_FANOUT_CBPF:
1632                 return fanout_set_data_cbpf(po, data, len);
1633         case PACKET_FANOUT_EBPF:
1634                 return fanout_set_data_ebpf(po, data, len);
1635         default:
1636                 return -EINVAL;
1637         }
1638 }
1639
1640 static void fanout_release_data(struct packet_fanout *f)
1641 {
1642         switch (f->type) {
1643         case PACKET_FANOUT_CBPF:
1644         case PACKET_FANOUT_EBPF:
1645                 __fanout_set_data_bpf(f, NULL);
1646         }
1647 }
1648
1649 static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
1650 {
1651         struct packet_fanout *f;
1652
1653         list_for_each_entry(f, &fanout_list, list) {
1654                 if (f->id == candidate_id &&
1655                     read_pnet(&f->net) == sock_net(sk)) {
1656                         return false;
1657                 }
1658         }
1659         return true;
1660 }
1661
1662 static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
1663 {
1664         u16 id = fanout_next_id;
1665
1666         do {
1667                 if (__fanout_id_is_free(sk, id)) {
1668                         *new_id = id;
1669                         fanout_next_id = id + 1;
1670                         return true;
1671                 }
1672
1673                 id++;
1674         } while (id != fanout_next_id);
1675
1676         return false;
1677 }
1678
1679 static int fanout_add(struct sock *sk, struct fanout_args *args)
1680 {
1681         struct packet_rollover *rollover = NULL;
1682         struct packet_sock *po = pkt_sk(sk);
1683         u16 type_flags = args->type_flags;
1684         struct packet_fanout *f, *match;
1685         u8 type = type_flags & 0xff;
1686         u8 flags = type_flags >> 8;
1687         u16 id = args->id;
1688         int err;
1689
1690         switch (type) {
1691         case PACKET_FANOUT_ROLLOVER:
1692                 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1693                         return -EINVAL;
1694                 break;
1695         case PACKET_FANOUT_HASH:
1696         case PACKET_FANOUT_LB:
1697         case PACKET_FANOUT_CPU:
1698         case PACKET_FANOUT_RND:
1699         case PACKET_FANOUT_QM:
1700         case PACKET_FANOUT_CBPF:
1701         case PACKET_FANOUT_EBPF:
1702                 break;
1703         default:
1704                 return -EINVAL;
1705         }
1706
1707         mutex_lock(&fanout_mutex);
1708
1709         err = -EALREADY;
1710         if (po->fanout)
1711                 goto out;
1712
1713         if (type == PACKET_FANOUT_ROLLOVER ||
1714             (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
1715                 err = -ENOMEM;
1716                 rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1717                 if (!rollover)
1718                         goto out;
1719                 atomic_long_set(&rollover->num, 0);
1720                 atomic_long_set(&rollover->num_huge, 0);
1721                 atomic_long_set(&rollover->num_failed, 0);
1722         }
1723
1724         if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
1725                 if (id != 0) {
1726                         err = -EINVAL;
1727                         goto out;
1728                 }
1729                 if (!fanout_find_new_id(sk, &id)) {
1730                         err = -ENOMEM;
1731                         goto out;
1732                 }
1733                 /* ephemeral flag for the first socket in the group: drop it */
1734                 flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
1735         }
1736
1737         match = NULL;
1738         list_for_each_entry(f, &fanout_list, list) {
1739                 if (f->id == id &&
1740                     read_pnet(&f->net) == sock_net(sk)) {
1741                         match = f;
1742                         break;
1743                 }
1744         }
1745         err = -EINVAL;
1746         if (match) {
1747                 if (match->flags != flags)
1748                         goto out;
1749                 if (args->max_num_members &&
1750                     args->max_num_members != match->max_num_members)
1751                         goto out;
1752         } else {
1753                 if (args->max_num_members > PACKET_FANOUT_MAX)
1754                         goto out;
1755                 if (!args->max_num_members)
1756                         /* legacy PACKET_FANOUT_MAX */
1757                         args->max_num_members = 256;
1758                 err = -ENOMEM;
1759                 match = kvzalloc(struct_size(match, arr, args->max_num_members),
1760                                  GFP_KERNEL);
1761                 if (!match)
1762                         goto out;
1763                 write_pnet(&match->net, sock_net(sk));
1764                 match->id = id;
1765                 match->type = type;
1766                 match->flags = flags;
1767                 INIT_LIST_HEAD(&match->list);
1768                 spin_lock_init(&match->lock);
1769                 refcount_set(&match->sk_ref, 0);
1770                 fanout_init_data(match);
1771                 match->prot_hook.type = po->prot_hook.type;
1772                 match->prot_hook.dev = po->prot_hook.dev;
1773                 match->prot_hook.func = packet_rcv_fanout;
1774                 match->prot_hook.af_packet_priv = match;
1775                 match->prot_hook.af_packet_net = read_pnet(&match->net);
1776                 match->prot_hook.id_match = match_fanout_group;
1777                 match->max_num_members = args->max_num_members;
1778                 match->prot_hook.ignore_outgoing = type_flags & PACKET_FANOUT_FLAG_IGNORE_OUTGOING;
1779                 list_add(&match->list, &fanout_list);
1780         }
1781         err = -EINVAL;
1782
1783         spin_lock(&po->bind_lock);
1784         if (packet_sock_flag(po, PACKET_SOCK_RUNNING) &&
1785             match->type == type &&
1786             match->prot_hook.type == po->prot_hook.type &&
1787             match->prot_hook.dev == po->prot_hook.dev) {
1788                 err = -ENOSPC;
1789                 if (refcount_read(&match->sk_ref) < match->max_num_members) {
1790                         __dev_remove_pack(&po->prot_hook);
1791
1792                         /* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
1793                         WRITE_ONCE(po->fanout, match);
1794
1795                         po->rollover = rollover;
1796                         rollover = NULL;
1797                         refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
1798                         __fanout_link(sk, po);
1799                         err = 0;
1800                 }
1801         }
1802         spin_unlock(&po->bind_lock);
1803
1804         if (err && !refcount_read(&match->sk_ref)) {
1805                 list_del(&match->list);
1806                 kvfree(match);
1807         }
1808
1809 out:
1810         kfree(rollover);
1811         mutex_unlock(&fanout_mutex);
1812         return err;
1813 }
1814
1815 /* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1816  * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1817  * It is the responsibility of the caller to call fanout_release_data() and
1818  * free the returned packet_fanout (after synchronize_net())
1819  */
1820 static struct packet_fanout *fanout_release(struct sock *sk)
1821 {
1822         struct packet_sock *po = pkt_sk(sk);
1823         struct packet_fanout *f;
1824
1825         mutex_lock(&fanout_mutex);
1826         f = po->fanout;
1827         if (f) {
1828                 po->fanout = NULL;
1829
1830                 if (refcount_dec_and_test(&f->sk_ref))
1831                         list_del(&f->list);
1832                 else
1833                         f = NULL;
1834         }
1835         mutex_unlock(&fanout_mutex);
1836
1837         return f;
1838 }
1839
1840 static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1841                                           struct sk_buff *skb)
1842 {
1843         /* Earlier code assumed this would be a VLAN pkt, double-check
1844          * this now that we have the actual packet in hand. We can only
1845          * do this check on Ethernet devices.
1846          */
1847         if (unlikely(dev->type != ARPHRD_ETHER))
1848                 return false;
1849
1850         skb_reset_mac_header(skb);
1851         return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1852 }
1853
1854 static const struct proto_ops packet_ops;
1855
1856 static const struct proto_ops packet_ops_spkt;
1857
1858 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1859                            struct packet_type *pt, struct net_device *orig_dev)
1860 {
1861         struct sock *sk;
1862         struct sockaddr_pkt *spkt;
1863
1864         /*
1865          *      When we registered the protocol we saved the socket in the data
1866          *      field for just this event.
1867          */
1868
1869         sk = pt->af_packet_priv;
1870
1871         /*
1872          *      Yank back the headers [hope the device set this
1873          *      right or kerboom...]
1874          *
1875          *      Incoming packets have ll header pulled,
1876          *      push it back.
1877          *
1878          *      For outgoing ones skb->data == skb_mac_header(skb)
1879          *      so that this procedure is noop.
1880          */
1881
1882         if (skb->pkt_type == PACKET_LOOPBACK)
1883                 goto out;
1884
1885         if (!net_eq(dev_net(dev), sock_net(sk)))
1886                 goto out;
1887
1888         skb = skb_share_check(skb, GFP_ATOMIC);
1889         if (skb == NULL)
1890                 goto oom;
1891
1892         /* drop any routing info */
1893         skb_dst_drop(skb);
1894
1895         /* drop conntrack reference */
1896         nf_reset_ct(skb);
1897
1898         spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1899
1900         skb_push(skb, skb->data - skb_mac_header(skb));
1901
1902         /*
1903          *      The SOCK_PACKET socket receives _all_ frames.
1904          */
1905
1906         spkt->spkt_family = dev->type;
1907         strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1908         spkt->spkt_protocol = skb->protocol;
1909
1910         /*
1911          *      Charge the memory to the socket. This is done specifically
1912          *      to prevent sockets using all the memory up.
1913          */
1914
1915         if (sock_queue_rcv_skb(sk, skb) == 0)
1916                 return 0;
1917
1918 out:
1919         kfree_skb(skb);
1920 oom:
1921         return 0;
1922 }
1923
1924 static void packet_parse_headers(struct sk_buff *skb, struct socket *sock)
1925 {
1926         int depth;
1927
1928         if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) &&
1929             sock->type == SOCK_RAW) {
1930                 skb_reset_mac_header(skb);
1931                 skb->protocol = dev_parse_header_protocol(skb);
1932         }
1933
1934         /* Move network header to the right position for VLAN tagged packets */
1935         if (likely(skb->dev->type == ARPHRD_ETHER) &&
1936             eth_type_vlan(skb->protocol) &&
1937             vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0)
1938                 skb_set_network_header(skb, depth);
1939
1940         skb_probe_transport_header(skb);
1941 }
1942
1943 /*
1944  *      Output a raw packet to a device layer. This bypasses all the other
1945  *      protocol layers and you must therefore supply it with a complete frame
1946  */
1947
1948 static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1949                                size_t len)
1950 {
1951         struct sock *sk = sock->sk;
1952         DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1953         struct sk_buff *skb = NULL;
1954         struct net_device *dev;
1955         struct sockcm_cookie sockc;
1956         __be16 proto = 0;
1957         int err;
1958         int extra_len = 0;
1959
1960         /*
1961          *      Get and verify the address.
1962          */
1963
1964         if (saddr) {
1965                 if (msg->msg_namelen < sizeof(struct sockaddr))
1966                         return -EINVAL;
1967                 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1968                         proto = saddr->spkt_protocol;
1969         } else
1970                 return -ENOTCONN;       /* SOCK_PACKET must be sent giving an address */
1971
1972         /*
1973          *      Find the device first to size check it
1974          */
1975
1976         saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1977 retry:
1978         rcu_read_lock();
1979         dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1980         err = -ENODEV;
1981         if (dev == NULL)
1982                 goto out_unlock;
1983
1984         err = -ENETDOWN;
1985         if (!(dev->flags & IFF_UP))
1986                 goto out_unlock;
1987
1988         /*
1989          * You may not queue a frame bigger than the mtu. This is the lowest level
1990          * raw protocol and you must do your own fragmentation at this level.
1991          */
1992
1993         if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1994                 if (!netif_supports_nofcs(dev)) {
1995                         err = -EPROTONOSUPPORT;
1996                         goto out_unlock;
1997                 }
1998                 extra_len = 4; /* We're doing our own CRC */
1999         }
2000
2001         err = -EMSGSIZE;
2002         if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
2003                 goto out_unlock;
2004
2005         if (!skb) {
2006                 size_t reserved = LL_RESERVED_SPACE(dev);
2007                 int tlen = dev->needed_tailroom;
2008                 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
2009
2010                 rcu_read_unlock();
2011                 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
2012                 if (skb == NULL)
2013                         return -ENOBUFS;
2014                 /* FIXME: Save some space for broken drivers that write a hard
2015                  * header at transmission time by themselves. PPP is the notable
2016                  * one here. This should really be fixed at the driver level.
2017                  */
2018                 skb_reserve(skb, reserved);
2019                 skb_reset_network_header(skb);
2020
2021                 /* Try to align data part correctly */
2022                 if (hhlen) {
2023                         skb->data -= hhlen;
2024                         skb->tail -= hhlen;
2025                         if (len < hhlen)
2026                                 skb_reset_network_header(skb);
2027                 }
2028                 err = memcpy_from_msg(skb_put(skb, len), msg, len);
2029                 if (err)
2030                         goto out_free;
2031                 goto retry;
2032         }
2033
2034         if (!dev_validate_header(dev, skb->data, len) || !skb->len) {
2035                 err = -EINVAL;
2036                 goto out_unlock;
2037         }
2038         if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
2039             !packet_extra_vlan_len_allowed(dev, skb)) {
2040                 err = -EMSGSIZE;
2041                 goto out_unlock;
2042         }
2043
2044         sockcm_init(&sockc, sk);
2045         if (msg->msg_controllen) {
2046                 err = sock_cmsg_send(sk, msg, &sockc);
2047                 if (unlikely(err))
2048                         goto out_unlock;
2049         }
2050
2051         skb->protocol = proto;
2052         skb->dev = dev;
2053         skb->priority = sk->sk_priority;
2054         skb->mark = sk->sk_mark;
2055         skb->tstamp = sockc.transmit_time;
2056
2057         skb_setup_tx_timestamp(skb, sockc.tsflags);
2058
2059         if (unlikely(extra_len == 4))
2060                 skb->no_fcs = 1;
2061
2062         packet_parse_headers(skb, sock);
2063
2064         dev_queue_xmit(skb);
2065         rcu_read_unlock();
2066         return len;
2067
2068 out_unlock:
2069         rcu_read_unlock();
2070 out_free:
2071         kfree_skb(skb);
2072         return err;
2073 }
2074
2075 static unsigned int run_filter(struct sk_buff *skb,
2076                                const struct sock *sk,
2077                                unsigned int res)
2078 {
2079         struct sk_filter *filter;
2080
2081         rcu_read_lock();
2082         filter = rcu_dereference(sk->sk_filter);
2083         if (filter != NULL)
2084                 res = bpf_prog_run_clear_cb(filter->prog, skb);
2085         rcu_read_unlock();
2086
2087         return res;
2088 }
2089
2090 static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
2091                            size_t *len, int vnet_hdr_sz)
2092 {
2093         struct virtio_net_hdr_mrg_rxbuf vnet_hdr = { .num_buffers = 0 };
2094
2095         if (*len < vnet_hdr_sz)
2096                 return -EINVAL;
2097         *len -= vnet_hdr_sz;
2098
2099         if (virtio_net_hdr_from_skb(skb, (struct virtio_net_hdr *)&vnet_hdr, vio_le(), true, 0))
2100                 return -EINVAL;
2101
2102         return memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_sz);
2103 }
2104
2105 /*
2106  * This function makes lazy skb cloning in hope that most of packets
2107  * are discarded by BPF.
2108  *
2109  * Note tricky part: we DO mangle shared skb! skb->data, skb->len
2110  * and skb->cb are mangled. It works because (and until) packets
2111  * falling here are owned by current CPU. Output packets are cloned
2112  * by dev_queue_xmit_nit(), input packets are processed by net_bh
2113  * sequentially, so that if we return skb to original state on exit,
2114  * we will not harm anyone.
2115  */
2116
2117 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2118                       struct packet_type *pt, struct net_device *orig_dev)
2119 {
2120         struct sock *sk;
2121         struct sockaddr_ll *sll;
2122         struct packet_sock *po;
2123         u8 *skb_head = skb->data;
2124         int skb_len = skb->len;
2125         unsigned int snaplen, res;
2126         bool is_drop_n_account = false;
2127
2128         if (skb->pkt_type == PACKET_LOOPBACK)
2129                 goto drop;
2130
2131         sk = pt->af_packet_priv;
2132         po = pkt_sk(sk);
2133
2134         if (!net_eq(dev_net(dev), sock_net(sk)))
2135                 goto drop;
2136
2137         skb->dev = dev;
2138
2139         if (dev_has_header(dev)) {
2140                 /* The device has an explicit notion of ll header,
2141                  * exported to higher levels.
2142                  *
2143                  * Otherwise, the device hides details of its frame
2144                  * structure, so that corresponding packet head is
2145                  * never delivered to user.
2146                  */
2147                 if (sk->sk_type != SOCK_DGRAM)
2148                         skb_push(skb, skb->data - skb_mac_header(skb));
2149                 else if (skb->pkt_type == PACKET_OUTGOING) {
2150                         /* Special case: outgoing packets have ll header at head */
2151                         skb_pull(skb, skb_network_offset(skb));
2152                 }
2153         }
2154
2155         snaplen = skb->len;
2156
2157         res = run_filter(skb, sk, snaplen);
2158         if (!res)
2159                 goto drop_n_restore;
2160         if (snaplen > res)
2161                 snaplen = res;
2162
2163         if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
2164                 goto drop_n_acct;
2165
2166         if (skb_shared(skb)) {
2167                 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2168                 if (nskb == NULL)
2169                         goto drop_n_acct;
2170
2171                 if (skb_head != skb->data) {
2172                         skb->data = skb_head;
2173                         skb->len = skb_len;
2174                 }
2175                 consume_skb(skb);
2176                 skb = nskb;
2177         }
2178
2179         sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
2180
2181         sll = &PACKET_SKB_CB(skb)->sa.ll;
2182         sll->sll_hatype = dev->type;
2183         sll->sll_pkttype = skb->pkt_type;
2184         if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2185                 sll->sll_ifindex = orig_dev->ifindex;
2186         else
2187                 sll->sll_ifindex = dev->ifindex;
2188
2189         sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2190
2191         /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2192          * Use their space for storing the original skb length.
2193          */
2194         PACKET_SKB_CB(skb)->sa.origlen = skb->len;
2195
2196         if (pskb_trim(skb, snaplen))
2197                 goto drop_n_acct;
2198
2199         skb_set_owner_r(skb, sk);
2200         skb->dev = NULL;
2201         skb_dst_drop(skb);
2202
2203         /* drop conntrack reference */
2204         nf_reset_ct(skb);
2205
2206         spin_lock(&sk->sk_receive_queue.lock);
2207         po->stats.stats1.tp_packets++;
2208         sock_skb_set_dropcount(sk, skb);
2209         skb_clear_delivery_time(skb);
2210         __skb_queue_tail(&sk->sk_receive_queue, skb);
2211         spin_unlock(&sk->sk_receive_queue.lock);
2212         sk->sk_data_ready(sk);
2213         return 0;
2214
2215 drop_n_acct:
2216         is_drop_n_account = true;
2217         atomic_inc(&po->tp_drops);
2218         atomic_inc(&sk->sk_drops);
2219
2220 drop_n_restore:
2221         if (skb_head != skb->data && skb_shared(skb)) {
2222                 skb->data = skb_head;
2223                 skb->len = skb_len;
2224         }
2225 drop:
2226         if (!is_drop_n_account)
2227                 consume_skb(skb);
2228         else
2229                 kfree_skb(skb);
2230         return 0;
2231 }
2232
2233 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2234                        struct packet_type *pt, struct net_device *orig_dev)
2235 {
2236         struct sock *sk;
2237         struct packet_sock *po;
2238         struct sockaddr_ll *sll;
2239         union tpacket_uhdr h;
2240         u8 *skb_head = skb->data;
2241         int skb_len = skb->len;
2242         unsigned int snaplen, res;
2243         unsigned long status = TP_STATUS_USER;
2244         unsigned short macoff, hdrlen;
2245         unsigned int netoff;
2246         struct sk_buff *copy_skb = NULL;
2247         struct timespec64 ts;
2248         __u32 ts_status;
2249         bool is_drop_n_account = false;
2250         unsigned int slot_id = 0;
2251         int vnet_hdr_sz = 0;
2252
2253         /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2254          * We may add members to them until current aligned size without forcing
2255          * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2256          */
2257         BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2258         BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2259
2260         if (skb->pkt_type == PACKET_LOOPBACK)
2261                 goto drop;
2262
2263         sk = pt->af_packet_priv;
2264         po = pkt_sk(sk);
2265
2266         if (!net_eq(dev_net(dev), sock_net(sk)))
2267                 goto drop;
2268
2269         if (dev_has_header(dev)) {
2270                 if (sk->sk_type != SOCK_DGRAM)
2271                         skb_push(skb, skb->data - skb_mac_header(skb));
2272                 else if (skb->pkt_type == PACKET_OUTGOING) {
2273                         /* Special case: outgoing packets have ll header at head */
2274                         skb_pull(skb, skb_network_offset(skb));
2275                 }
2276         }
2277
2278         snaplen = skb->len;
2279
2280         res = run_filter(skb, sk, snaplen);
2281         if (!res)
2282                 goto drop_n_restore;
2283
2284         /* If we are flooded, just give up */
2285         if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
2286                 atomic_inc(&po->tp_drops);
2287                 goto drop_n_restore;
2288         }
2289
2290         if (skb->ip_summed == CHECKSUM_PARTIAL)
2291                 status |= TP_STATUS_CSUMNOTREADY;
2292         else if (skb->pkt_type != PACKET_OUTGOING &&
2293                  skb_csum_unnecessary(skb))
2294                 status |= TP_STATUS_CSUM_VALID;
2295         if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
2296                 status |= TP_STATUS_GSO_TCP;
2297
2298         if (snaplen > res)
2299                 snaplen = res;
2300
2301         if (sk->sk_type == SOCK_DGRAM) {
2302                 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2303                                   po->tp_reserve;
2304         } else {
2305                 unsigned int maclen = skb_network_offset(skb);
2306                 netoff = TPACKET_ALIGN(po->tp_hdrlen +
2307                                        (maclen < 16 ? 16 : maclen)) +
2308                                        po->tp_reserve;
2309                 vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2310                 if (vnet_hdr_sz)
2311                         netoff += vnet_hdr_sz;
2312                 macoff = netoff - maclen;
2313         }
2314         if (netoff > USHRT_MAX) {
2315                 atomic_inc(&po->tp_drops);
2316                 goto drop_n_restore;
2317         }
2318         if (po->tp_version <= TPACKET_V2) {
2319                 if (macoff + snaplen > po->rx_ring.frame_size) {
2320                         if (po->copy_thresh &&
2321                             atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
2322                                 if (skb_shared(skb)) {
2323                                         copy_skb = skb_clone(skb, GFP_ATOMIC);
2324                                 } else {
2325                                         copy_skb = skb_get(skb);
2326                                         skb_head = skb->data;
2327                                 }
2328                                 if (copy_skb) {
2329                                         memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
2330                                                sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
2331                                         skb_set_owner_r(copy_skb, sk);
2332                                 }
2333                         }
2334                         snaplen = po->rx_ring.frame_size - macoff;
2335                         if ((int)snaplen < 0) {
2336                                 snaplen = 0;
2337                                 vnet_hdr_sz = 0;
2338                         }
2339                 }
2340         } else if (unlikely(macoff + snaplen >
2341                             GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2342                 u32 nval;
2343
2344                 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2345                 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2346                             snaplen, nval, macoff);
2347                 snaplen = nval;
2348                 if (unlikely((int)snaplen < 0)) {
2349                         snaplen = 0;
2350                         macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2351                         vnet_hdr_sz = 0;
2352                 }
2353         }
2354         spin_lock(&sk->sk_receive_queue.lock);
2355         h.raw = packet_current_rx_frame(po, skb,
2356                                         TP_STATUS_KERNEL, (macoff+snaplen));
2357         if (!h.raw)
2358                 goto drop_n_account;
2359
2360         if (po->tp_version <= TPACKET_V2) {
2361                 slot_id = po->rx_ring.head;
2362                 if (test_bit(slot_id, po->rx_ring.rx_owner_map))
2363                         goto drop_n_account;
2364                 __set_bit(slot_id, po->rx_ring.rx_owner_map);
2365         }
2366
2367         if (vnet_hdr_sz &&
2368             virtio_net_hdr_from_skb(skb, h.raw + macoff -
2369                                     sizeof(struct virtio_net_hdr),
2370                                     vio_le(), true, 0)) {
2371                 if (po->tp_version == TPACKET_V3)
2372                         prb_clear_blk_fill_status(&po->rx_ring);
2373                 goto drop_n_account;
2374         }
2375
2376         if (po->tp_version <= TPACKET_V2) {
2377                 packet_increment_rx_head(po, &po->rx_ring);
2378         /*
2379          * LOSING will be reported till you read the stats,
2380          * because it's COR - Clear On Read.
2381          * Anyways, moving it for V1/V2 only as V3 doesn't need this
2382          * at packet level.
2383          */
2384                 if (atomic_read(&po->tp_drops))
2385                         status |= TP_STATUS_LOSING;
2386         }
2387
2388         po->stats.stats1.tp_packets++;
2389         if (copy_skb) {
2390                 status |= TP_STATUS_COPY;
2391                 skb_clear_delivery_time(copy_skb);
2392                 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2393         }
2394         spin_unlock(&sk->sk_receive_queue.lock);
2395
2396         skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
2397
2398         /* Always timestamp; prefer an existing software timestamp taken
2399          * closer to the time of capture.
2400          */
2401         ts_status = tpacket_get_timestamp(skb, &ts,
2402                                           READ_ONCE(po->tp_tstamp) |
2403                                           SOF_TIMESTAMPING_SOFTWARE);
2404         if (!ts_status)
2405                 ktime_get_real_ts64(&ts);
2406
2407         status |= ts_status;
2408
2409         switch (po->tp_version) {
2410         case TPACKET_V1:
2411                 h.h1->tp_len = skb->len;
2412                 h.h1->tp_snaplen = snaplen;
2413                 h.h1->tp_mac = macoff;
2414                 h.h1->tp_net = netoff;
2415                 h.h1->tp_sec = ts.tv_sec;
2416                 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
2417                 hdrlen = sizeof(*h.h1);
2418                 break;
2419         case TPACKET_V2:
2420                 h.h2->tp_len = skb->len;
2421                 h.h2->tp_snaplen = snaplen;
2422                 h.h2->tp_mac = macoff;
2423                 h.h2->tp_net = netoff;
2424                 h.h2->tp_sec = ts.tv_sec;
2425                 h.h2->tp_nsec = ts.tv_nsec;
2426                 if (skb_vlan_tag_present(skb)) {
2427                         h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
2428                         h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2429                         status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2430                 } else {
2431                         h.h2->tp_vlan_tci = 0;
2432                         h.h2->tp_vlan_tpid = 0;
2433                 }
2434                 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
2435                 hdrlen = sizeof(*h.h2);
2436                 break;
2437         case TPACKET_V3:
2438                 /* tp_nxt_offset,vlan are already populated above.
2439                  * So DONT clear those fields here
2440                  */
2441                 h.h3->tp_status |= status;
2442                 h.h3->tp_len = skb->len;
2443                 h.h3->tp_snaplen = snaplen;
2444                 h.h3->tp_mac = macoff;
2445                 h.h3->tp_net = netoff;
2446                 h.h3->tp_sec  = ts.tv_sec;
2447                 h.h3->tp_nsec = ts.tv_nsec;
2448                 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
2449                 hdrlen = sizeof(*h.h3);
2450                 break;
2451         default:
2452                 BUG();
2453         }
2454
2455         sll = h.raw + TPACKET_ALIGN(hdrlen);
2456         sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2457         sll->sll_family = AF_PACKET;
2458         sll->sll_hatype = dev->type;
2459         sll->sll_protocol = skb->protocol;
2460         sll->sll_pkttype = skb->pkt_type;
2461         if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2462                 sll->sll_ifindex = orig_dev->ifindex;
2463         else
2464                 sll->sll_ifindex = dev->ifindex;
2465
2466         smp_mb();
2467
2468 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
2469         if (po->tp_version <= TPACKET_V2) {
2470                 u8 *start, *end;
2471
2472                 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2473                                         macoff + snaplen);
2474
2475                 for (start = h.raw; start < end; start += PAGE_SIZE)
2476                         flush_dcache_page(pgv_to_page(start));
2477         }
2478         smp_wmb();
2479 #endif
2480
2481         if (po->tp_version <= TPACKET_V2) {
2482                 spin_lock(&sk->sk_receive_queue.lock);
2483                 __packet_set_status(po, h.raw, status);
2484                 __clear_bit(slot_id, po->rx_ring.rx_owner_map);
2485                 spin_unlock(&sk->sk_receive_queue.lock);
2486                 sk->sk_data_ready(sk);
2487         } else if (po->tp_version == TPACKET_V3) {
2488                 prb_clear_blk_fill_status(&po->rx_ring);
2489         }
2490
2491 drop_n_restore:
2492         if (skb_head != skb->data && skb_shared(skb)) {
2493                 skb->data = skb_head;
2494                 skb->len = skb_len;
2495         }
2496 drop:
2497         if (!is_drop_n_account)
2498                 consume_skb(skb);
2499         else
2500                 kfree_skb(skb);
2501         return 0;
2502
2503 drop_n_account:
2504         spin_unlock(&sk->sk_receive_queue.lock);
2505         atomic_inc(&po->tp_drops);
2506         is_drop_n_account = true;
2507
2508         sk->sk_data_ready(sk);
2509         kfree_skb(copy_skb);
2510         goto drop_n_restore;
2511 }
2512
2513 static void tpacket_destruct_skb(struct sk_buff *skb)
2514 {
2515         struct packet_sock *po = pkt_sk(skb->sk);
2516
2517         if (likely(po->tx_ring.pg_vec)) {
2518                 void *ph;
2519                 __u32 ts;
2520
2521                 ph = skb_zcopy_get_nouarg(skb);
2522                 packet_dec_pending(&po->tx_ring);
2523
2524                 ts = __packet_set_timestamp(po, ph, skb);
2525                 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2526
2527                 if (!packet_read_pending(&po->tx_ring))
2528                         complete(&po->skb_completion);
2529         }
2530
2531         sock_wfree(skb);
2532 }
2533
2534 static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2535 {
2536         if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2537             (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2538              __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2539               __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2540                 vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2541                          __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2542                         __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2543
2544         if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2545                 return -EINVAL;
2546
2547         return 0;
2548 }
2549
2550 static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2551                                  struct virtio_net_hdr *vnet_hdr, int vnet_hdr_sz)
2552 {
2553         int ret;
2554
2555         if (*len < vnet_hdr_sz)
2556                 return -EINVAL;
2557         *len -= vnet_hdr_sz;
2558
2559         if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
2560                 return -EFAULT;
2561
2562         ret = __packet_snd_vnet_parse(vnet_hdr, *len);
2563         if (ret)
2564                 return ret;
2565
2566         /* move iter to point to the start of mac header */
2567         if (vnet_hdr_sz != sizeof(struct virtio_net_hdr))
2568                 iov_iter_advance(&msg->msg_iter, vnet_hdr_sz - sizeof(struct virtio_net_hdr));
2569
2570         return 0;
2571 }
2572
2573 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2574                 void *frame, struct net_device *dev, void *data, int tp_len,
2575                 __be16 proto, unsigned char *addr, int hlen, int copylen,
2576                 const struct sockcm_cookie *sockc)
2577 {
2578         union tpacket_uhdr ph;
2579         int to_write, offset, len, nr_frags, len_max;
2580         struct socket *sock = po->sk.sk_socket;
2581         struct page *page;
2582         int err;
2583
2584         ph.raw = frame;
2585
2586         skb->protocol = proto;
2587         skb->dev = dev;
2588         skb->priority = po->sk.sk_priority;
2589         skb->mark = po->sk.sk_mark;
2590         skb->tstamp = sockc->transmit_time;
2591         skb_setup_tx_timestamp(skb, sockc->tsflags);
2592         skb_zcopy_set_nouarg(skb, ph.raw);
2593
2594         skb_reserve(skb, hlen);
2595         skb_reset_network_header(skb);
2596
2597         to_write = tp_len;
2598
2599         if (sock->type == SOCK_DGRAM) {
2600                 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2601                                 NULL, tp_len);
2602                 if (unlikely(err < 0))
2603                         return -EINVAL;
2604         } else if (copylen) {
2605                 int hdrlen = min_t(int, copylen, tp_len);
2606
2607                 skb_push(skb, dev->hard_header_len);
2608                 skb_put(skb, copylen - dev->hard_header_len);
2609                 err = skb_store_bits(skb, 0, data, hdrlen);
2610                 if (unlikely(err))
2611                         return err;
2612                 if (!dev_validate_header(dev, skb->data, hdrlen))
2613                         return -EINVAL;
2614
2615                 data += hdrlen;
2616                 to_write -= hdrlen;
2617         }
2618
2619         offset = offset_in_page(data);
2620         len_max = PAGE_SIZE - offset;
2621         len = ((to_write > len_max) ? len_max : to_write);
2622
2623         skb->data_len = to_write;
2624         skb->len += to_write;
2625         skb->truesize += to_write;
2626         refcount_add(to_write, &po->sk.sk_wmem_alloc);
2627
2628         while (likely(to_write)) {
2629                 nr_frags = skb_shinfo(skb)->nr_frags;
2630
2631                 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2632                         pr_err("Packet exceed the number of skb frags(%u)\n",
2633                                (unsigned int)MAX_SKB_FRAGS);
2634                         return -EFAULT;
2635                 }
2636
2637                 page = pgv_to_page(data);
2638                 data += len;
2639                 flush_dcache_page(page);
2640                 get_page(page);
2641                 skb_fill_page_desc(skb, nr_frags, page, offset, len);
2642                 to_write -= len;
2643                 offset = 0;
2644                 len_max = PAGE_SIZE;
2645                 len = ((to_write > len_max) ? len_max : to_write);
2646         }
2647
2648         packet_parse_headers(skb, sock);
2649
2650         return tp_len;
2651 }
2652
2653 static int tpacket_parse_header(struct packet_sock *po, void *frame,
2654                                 int size_max, void **data)
2655 {
2656         union tpacket_uhdr ph;
2657         int tp_len, off;
2658
2659         ph.raw = frame;
2660
2661         switch (po->tp_version) {
2662         case TPACKET_V3:
2663                 if (ph.h3->tp_next_offset != 0) {
2664                         pr_warn_once("variable sized slot not supported");
2665                         return -EINVAL;
2666                 }
2667                 tp_len = ph.h3->tp_len;
2668                 break;
2669         case TPACKET_V2:
2670                 tp_len = ph.h2->tp_len;
2671                 break;
2672         default:
2673                 tp_len = ph.h1->tp_len;
2674                 break;
2675         }
2676         if (unlikely(tp_len > size_max)) {
2677                 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2678                 return -EMSGSIZE;
2679         }
2680
2681         if (unlikely(packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF))) {
2682                 int off_min, off_max;
2683
2684                 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2685                 off_max = po->tx_ring.frame_size - tp_len;
2686                 if (po->sk.sk_type == SOCK_DGRAM) {
2687                         switch (po->tp_version) {
2688                         case TPACKET_V3:
2689                                 off = ph.h3->tp_net;
2690                                 break;
2691                         case TPACKET_V2:
2692                                 off = ph.h2->tp_net;
2693                                 break;
2694                         default:
2695                                 off = ph.h1->tp_net;
2696                                 break;
2697                         }
2698                 } else {
2699                         switch (po->tp_version) {
2700                         case TPACKET_V3:
2701                                 off = ph.h3->tp_mac;
2702                                 break;
2703                         case TPACKET_V2:
2704                                 off = ph.h2->tp_mac;
2705                                 break;
2706                         default:
2707                                 off = ph.h1->tp_mac;
2708                                 break;
2709                         }
2710                 }
2711                 if (unlikely((off < off_min) || (off_max < off)))
2712                         return -EINVAL;
2713         } else {
2714                 off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2715         }
2716
2717         *data = frame + off;
2718         return tp_len;
2719 }
2720
2721 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2722 {
2723         struct sk_buff *skb = NULL;
2724         struct net_device *dev;
2725         struct virtio_net_hdr *vnet_hdr = NULL;
2726         struct sockcm_cookie sockc;
2727         __be16 proto;
2728         int err, reserve = 0;
2729         void *ph;
2730         DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2731         bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2732         int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2733         unsigned char *addr = NULL;
2734         int tp_len, size_max;
2735         void *data;
2736         int len_sum = 0;
2737         int status = TP_STATUS_AVAILABLE;
2738         int hlen, tlen, copylen = 0;
2739         long timeo = 0;
2740
2741         mutex_lock(&po->pg_vec_lock);
2742
2743         /* packet_sendmsg() check on tx_ring.pg_vec was lockless,
2744          * we need to confirm it under protection of pg_vec_lock.
2745          */
2746         if (unlikely(!po->tx_ring.pg_vec)) {
2747                 err = -EBUSY;
2748                 goto out;
2749         }
2750         if (likely(saddr == NULL)) {
2751                 dev     = packet_cached_dev_get(po);
2752                 proto   = READ_ONCE(po->num);
2753         } else {
2754                 err = -EINVAL;
2755                 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2756                         goto out;
2757                 if (msg->msg_namelen < (saddr->sll_halen
2758                                         + offsetof(struct sockaddr_ll,
2759                                                 sll_addr)))
2760                         goto out;
2761                 proto   = saddr->sll_protocol;
2762                 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2763                 if (po->sk.sk_socket->type == SOCK_DGRAM) {
2764                         if (dev && msg->msg_namelen < dev->addr_len +
2765                                    offsetof(struct sockaddr_ll, sll_addr))
2766                                 goto out_put;
2767                         addr = saddr->sll_addr;
2768                 }
2769         }
2770
2771         err = -ENXIO;
2772         if (unlikely(dev == NULL))
2773                 goto out;
2774         err = -ENETDOWN;
2775         if (unlikely(!(dev->flags & IFF_UP)))
2776                 goto out_put;
2777
2778         sockcm_init(&sockc, &po->sk);
2779         if (msg->msg_controllen) {
2780                 err = sock_cmsg_send(&po->sk, msg, &sockc);
2781                 if (unlikely(err))
2782                         goto out_put;
2783         }
2784
2785         if (po->sk.sk_socket->type == SOCK_RAW)
2786                 reserve = dev->hard_header_len;
2787         size_max = po->tx_ring.frame_size
2788                 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2789
2790         if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !vnet_hdr_sz)
2791                 size_max = dev->mtu + reserve + VLAN_HLEN;
2792
2793         reinit_completion(&po->skb_completion);
2794
2795         do {
2796                 ph = packet_current_frame(po, &po->tx_ring,
2797                                           TP_STATUS_SEND_REQUEST);
2798                 if (unlikely(ph == NULL)) {
2799                         if (need_wait && skb) {
2800                                 timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT);
2801                                 timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo);
2802                                 if (timeo <= 0) {
2803                                         err = !timeo ? -ETIMEDOUT : -ERESTARTSYS;
2804                                         goto out_put;
2805                                 }
2806                         }
2807                         /* check for additional frames */
2808                         continue;
2809                 }
2810
2811                 skb = NULL;
2812                 tp_len = tpacket_parse_header(po, ph, size_max, &data);
2813                 if (tp_len < 0)
2814                         goto tpacket_error;
2815
2816                 status = TP_STATUS_SEND_REQUEST;
2817                 hlen = LL_RESERVED_SPACE(dev);
2818                 tlen = dev->needed_tailroom;
2819                 if (vnet_hdr_sz) {
2820                         vnet_hdr = data;
2821                         data += vnet_hdr_sz;
2822                         tp_len -= vnet_hdr_sz;
2823                         if (tp_len < 0 ||
2824                             __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2825                                 tp_len = -EINVAL;
2826                                 goto tpacket_error;
2827                         }
2828                         copylen = __virtio16_to_cpu(vio_le(),
2829                                                     vnet_hdr->hdr_len);
2830                 }
2831                 copylen = max_t(int, copylen, dev->hard_header_len);
2832                 skb = sock_alloc_send_skb(&po->sk,
2833                                 hlen + tlen + sizeof(struct sockaddr_ll) +
2834                                 (copylen - dev->hard_header_len),
2835                                 !need_wait, &err);
2836
2837                 if (unlikely(skb == NULL)) {
2838                         /* we assume the socket was initially writeable ... */
2839                         if (likely(len_sum > 0))
2840                                 err = len_sum;
2841                         goto out_status;
2842                 }
2843                 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
2844                                           addr, hlen, copylen, &sockc);
2845                 if (likely(tp_len >= 0) &&
2846                     tp_len > dev->mtu + reserve &&
2847                     !vnet_hdr_sz &&
2848                     !packet_extra_vlan_len_allowed(dev, skb))
2849                         tp_len = -EMSGSIZE;
2850
2851                 if (unlikely(tp_len < 0)) {
2852 tpacket_error:
2853                         if (packet_sock_flag(po, PACKET_SOCK_TP_LOSS)) {
2854                                 __packet_set_status(po, ph,
2855                                                 TP_STATUS_AVAILABLE);
2856                                 packet_increment_head(&po->tx_ring);
2857                                 kfree_skb(skb);
2858                                 continue;
2859                         } else {
2860                                 status = TP_STATUS_WRONG_FORMAT;
2861                                 err = tp_len;
2862                                 goto out_status;
2863                         }
2864                 }
2865
2866                 if (vnet_hdr_sz) {
2867                         if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) {
2868                                 tp_len = -EINVAL;
2869                                 goto tpacket_error;
2870                         }
2871                         virtio_net_hdr_set_proto(skb, vnet_hdr);
2872                 }
2873
2874                 skb->destructor = tpacket_destruct_skb;
2875                 __packet_set_status(po, ph, TP_STATUS_SENDING);
2876                 packet_inc_pending(&po->tx_ring);
2877
2878                 status = TP_STATUS_SEND_REQUEST;
2879                 err = packet_xmit(po, skb);
2880                 if (unlikely(err != 0)) {
2881                         if (err > 0)
2882                                 err = net_xmit_errno(err);
2883                         if (err && __packet_get_status(po, ph) ==
2884                                    TP_STATUS_AVAILABLE) {
2885                                 /* skb was destructed already */
2886                                 skb = NULL;
2887                                 goto out_status;
2888                         }
2889                         /*
2890                          * skb was dropped but not destructed yet;
2891                          * let's treat it like congestion or err < 0
2892                          */
2893                         err = 0;
2894                 }
2895                 packet_increment_head(&po->tx_ring);
2896                 len_sum += tp_len;
2897         } while (likely((ph != NULL) ||
2898                 /* Note: packet_read_pending() might be slow if we have
2899                  * to call it as it's per_cpu variable, but in fast-path
2900                  * we already short-circuit the loop with the first
2901                  * condition, and luckily don't have to go that path
2902                  * anyway.
2903                  */
2904                  (need_wait && packet_read_pending(&po->tx_ring))));
2905
2906         err = len_sum;
2907         goto out_put;
2908
2909 out_status:
2910         __packet_set_status(po, ph, status);
2911         kfree_skb(skb);
2912 out_put:
2913         dev_put(dev);
2914 out:
2915         mutex_unlock(&po->pg_vec_lock);
2916         return err;
2917 }
2918
2919 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2920                                         size_t reserve, size_t len,
2921                                         size_t linear, int noblock,
2922                                         int *err)
2923 {
2924         struct sk_buff *skb;
2925
2926         /* Under a page?  Don't bother with paged skb. */
2927         if (prepad + len < PAGE_SIZE || !linear)
2928                 linear = len;
2929
2930         skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2931                                    err, 0);
2932         if (!skb)
2933                 return NULL;
2934
2935         skb_reserve(skb, reserve);
2936         skb_put(skb, linear);
2937         skb->data_len = len - linear;
2938         skb->len += len - linear;
2939
2940         return skb;
2941 }
2942
2943 static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2944 {
2945         struct sock *sk = sock->sk;
2946         DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2947         struct sk_buff *skb;
2948         struct net_device *dev;
2949         __be16 proto;
2950         unsigned char *addr = NULL;
2951         int err, reserve = 0;
2952         struct sockcm_cookie sockc;
2953         struct virtio_net_hdr vnet_hdr = { 0 };
2954         int offset = 0;
2955         struct packet_sock *po = pkt_sk(sk);
2956         int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2957         int hlen, tlen, linear;
2958         int extra_len = 0;
2959
2960         /*
2961          *      Get and verify the address.
2962          */
2963
2964         if (likely(saddr == NULL)) {
2965                 dev     = packet_cached_dev_get(po);
2966                 proto   = READ_ONCE(po->num);
2967         } else {
2968                 err = -EINVAL;
2969                 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2970                         goto out;
2971                 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2972                         goto out;
2973                 proto   = saddr->sll_protocol;
2974                 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2975                 if (sock->type == SOCK_DGRAM) {
2976                         if (dev && msg->msg_namelen < dev->addr_len +
2977                                    offsetof(struct sockaddr_ll, sll_addr))
2978                                 goto out_unlock;
2979                         addr = saddr->sll_addr;
2980                 }
2981         }
2982
2983         err = -ENXIO;
2984         if (unlikely(dev == NULL))
2985                 goto out_unlock;
2986         err = -ENETDOWN;
2987         if (unlikely(!(dev->flags & IFF_UP)))
2988                 goto out_unlock;
2989
2990         sockcm_init(&sockc, sk);
2991         sockc.mark = sk->sk_mark;
2992         if (msg->msg_controllen) {
2993                 err = sock_cmsg_send(sk, msg, &sockc);
2994                 if (unlikely(err))
2995                         goto out_unlock;
2996         }
2997
2998         if (sock->type == SOCK_RAW)
2999                 reserve = dev->hard_header_len;
3000         if (vnet_hdr_sz) {
3001                 err = packet_snd_vnet_parse(msg, &len, &vnet_hdr, vnet_hdr_sz);
3002                 if (err)
3003                         goto out_unlock;
3004         }
3005
3006         if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
3007                 if (!netif_supports_nofcs(dev)) {
3008                         err = -EPROTONOSUPPORT;
3009                         goto out_unlock;
3010                 }
3011                 extra_len = 4; /* We're doing our own CRC */
3012         }
3013
3014         err = -EMSGSIZE;
3015         if (!vnet_hdr.gso_type &&
3016             (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
3017                 goto out_unlock;
3018
3019         err = -ENOBUFS;
3020         hlen = LL_RESERVED_SPACE(dev);
3021         tlen = dev->needed_tailroom;
3022         linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
3023         linear = max(linear, min_t(int, len, dev->hard_header_len));
3024         skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
3025                                msg->msg_flags & MSG_DONTWAIT, &err);
3026         if (skb == NULL)
3027                 goto out_unlock;
3028
3029         skb_reset_network_header(skb);
3030
3031         err = -EINVAL;
3032         if (sock->type == SOCK_DGRAM) {
3033                 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
3034                 if (unlikely(offset < 0))
3035                         goto out_free;
3036         } else if (reserve) {
3037                 skb_reserve(skb, -reserve);
3038                 if (len < reserve + sizeof(struct ipv6hdr) &&
3039                     dev->min_header_len != dev->hard_header_len)
3040                         skb_reset_network_header(skb);
3041         }
3042
3043         /* Returns -EFAULT on error */
3044         err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
3045         if (err)
3046                 goto out_free;
3047
3048         if ((sock->type == SOCK_RAW &&
3049              !dev_validate_header(dev, skb->data, len)) || !skb->len) {
3050                 err = -EINVAL;
3051                 goto out_free;
3052         }
3053
3054         skb_setup_tx_timestamp(skb, sockc.tsflags);
3055
3056         if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
3057             !packet_extra_vlan_len_allowed(dev, skb)) {
3058                 err = -EMSGSIZE;
3059                 goto out_free;
3060         }
3061
3062         skb->protocol = proto;
3063         skb->dev = dev;
3064         skb->priority = sk->sk_priority;
3065         skb->mark = sockc.mark;
3066         skb->tstamp = sockc.transmit_time;
3067
3068         if (unlikely(extra_len == 4))
3069                 skb->no_fcs = 1;
3070
3071         packet_parse_headers(skb, sock);
3072
3073         if (vnet_hdr_sz) {
3074                 err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
3075                 if (err)
3076                         goto out_free;
3077                 len += vnet_hdr_sz;
3078                 virtio_net_hdr_set_proto(skb, &vnet_hdr);
3079         }
3080
3081         err = packet_xmit(po, skb);
3082
3083         if (unlikely(err != 0)) {
3084                 if (err > 0)
3085                         err = net_xmit_errno(err);
3086                 if (err)
3087                         goto out_unlock;
3088         }
3089
3090         dev_put(dev);
3091
3092         return len;
3093
3094 out_free:
3095         kfree_skb(skb);
3096 out_unlock:
3097         dev_put(dev);
3098 out:
3099         return err;
3100 }
3101
3102 static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
3103 {
3104         struct sock *sk = sock->sk;
3105         struct packet_sock *po = pkt_sk(sk);
3106
3107         /* Reading tx_ring.pg_vec without holding pg_vec_lock is racy.
3108          * tpacket_snd() will redo the check safely.
3109          */
3110         if (data_race(po->tx_ring.pg_vec))
3111                 return tpacket_snd(po, msg);
3112
3113         return packet_snd(sock, msg, len);
3114 }
3115
3116 /*
3117  *      Close a PACKET socket. This is fairly simple. We immediately go
3118  *      to 'closed' state and remove our protocol entry in the device list.
3119  */
3120
3121 static int packet_release(struct socket *sock)
3122 {
3123         struct sock *sk = sock->sk;
3124         struct packet_sock *po;
3125         struct packet_fanout *f;
3126         struct net *net;
3127         union tpacket_req_u req_u;
3128
3129         if (!sk)
3130                 return 0;
3131
3132         net = sock_net(sk);
3133         po = pkt_sk(sk);
3134
3135         mutex_lock(&net->packet.sklist_lock);
3136         sk_del_node_init_rcu(sk);
3137         mutex_unlock(&net->packet.sklist_lock);
3138
3139         sock_prot_inuse_add(net, sk->sk_prot, -1);
3140
3141         spin_lock(&po->bind_lock);
3142         unregister_prot_hook(sk, false);
3143         packet_cached_dev_reset(po);
3144
3145         if (po->prot_hook.dev) {
3146                 netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
3147                 po->prot_hook.dev = NULL;
3148         }
3149         spin_unlock(&po->bind_lock);
3150
3151         packet_flush_mclist(sk);
3152
3153         lock_sock(sk);
3154         if (po->rx_ring.pg_vec) {
3155                 memset(&req_u, 0, sizeof(req_u));
3156                 packet_set_ring(sk, &req_u, 1, 0);
3157         }
3158
3159         if (po->tx_ring.pg_vec) {
3160                 memset(&req_u, 0, sizeof(req_u));
3161                 packet_set_ring(sk, &req_u, 1, 1);
3162         }
3163         release_sock(sk);
3164
3165         f = fanout_release(sk);
3166
3167         synchronize_net();
3168
3169         kfree(po->rollover);
3170         if (f) {
3171                 fanout_release_data(f);
3172                 kvfree(f);
3173         }
3174         /*
3175          *      Now the socket is dead. No more input will appear.
3176          */
3177         sock_orphan(sk);
3178         sock->sk = NULL;
3179
3180         /* Purge queues */
3181
3182         skb_queue_purge(&sk->sk_receive_queue);
3183         packet_free_pending(po);
3184
3185         sock_put(sk);
3186         return 0;
3187 }
3188
3189 /*
3190  *      Attach a packet hook.
3191  */
3192
3193 static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3194                           __be16 proto)
3195 {
3196         struct packet_sock *po = pkt_sk(sk);
3197         struct net_device *dev = NULL;
3198         bool unlisted = false;
3199         bool need_rehook;
3200         int ret = 0;
3201
3202         lock_sock(sk);
3203         spin_lock(&po->bind_lock);
3204         if (!proto)
3205                 proto = po->num;
3206
3207         rcu_read_lock();
3208
3209         if (po->fanout) {
3210                 ret = -EINVAL;
3211                 goto out_unlock;
3212         }
3213
3214         if (name) {
3215                 dev = dev_get_by_name_rcu(sock_net(sk), name);
3216                 if (!dev) {
3217                         ret = -ENODEV;
3218                         goto out_unlock;
3219                 }
3220         } else if (ifindex) {
3221                 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3222                 if (!dev) {
3223                         ret = -ENODEV;
3224                         goto out_unlock;
3225                 }
3226         }
3227
3228         need_rehook = po->prot_hook.type != proto || po->prot_hook.dev != dev;
3229
3230         if (need_rehook) {
3231                 dev_hold(dev);
3232                 if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
3233                         rcu_read_unlock();
3234                         /* prevents packet_notifier() from calling
3235                          * register_prot_hook()
3236                          */
3237                         WRITE_ONCE(po->num, 0);
3238                         __unregister_prot_hook(sk, true);
3239                         rcu_read_lock();
3240                         if (dev)
3241                                 unlisted = !dev_get_by_index_rcu(sock_net(sk),
3242                                                                  dev->ifindex);
3243                 }
3244
3245                 BUG_ON(packet_sock_flag(po, PACKET_SOCK_RUNNING));
3246                 WRITE_ONCE(po->num, proto);
3247                 po->prot_hook.type = proto;
3248
3249                 netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
3250
3251                 if (unlikely(unlisted)) {
3252                         po->prot_hook.dev = NULL;
3253                         WRITE_ONCE(po->ifindex, -1);
3254                         packet_cached_dev_reset(po);
3255                 } else {
3256                         netdev_hold(dev, &po->prot_hook.dev_tracker,
3257                                     GFP_ATOMIC);
3258                         po->prot_hook.dev = dev;
3259                         WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0);
3260                         packet_cached_dev_assign(po, dev);
3261                 }
3262                 dev_put(dev);
3263         }
3264
3265         if (proto == 0 || !need_rehook)
3266                 goto out_unlock;
3267
3268         if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
3269                 register_prot_hook(sk);
3270         } else {
3271                 sk->sk_err = ENETDOWN;
3272                 if (!sock_flag(sk, SOCK_DEAD))
3273                         sk_error_report(sk);
3274         }
3275
3276 out_unlock:
3277         rcu_read_unlock();
3278         spin_unlock(&po->bind_lock);
3279         release_sock(sk);
3280         return ret;
3281 }
3282
3283 /*
3284  *      Bind a packet socket to a device
3285  */
3286
3287 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3288                             int addr_len)
3289 {
3290         struct sock *sk = sock->sk;
3291         char name[sizeof(uaddr->sa_data_min) + 1];
3292
3293         /*
3294          *      Check legality
3295          */
3296
3297         if (addr_len != sizeof(struct sockaddr))
3298                 return -EINVAL;
3299         /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3300          * zero-terminated.
3301          */
3302         memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data_min));
3303         name[sizeof(uaddr->sa_data_min)] = 0;
3304
3305         return packet_do_bind(sk, name, 0, 0);
3306 }
3307
3308 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3309 {
3310         struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3311         struct sock *sk = sock->sk;
3312
3313         /*
3314          *      Check legality
3315          */
3316
3317         if (addr_len < sizeof(struct sockaddr_ll))
3318                 return -EINVAL;
3319         if (sll->sll_family != AF_PACKET)
3320                 return -EINVAL;
3321
3322         return packet_do_bind(sk, NULL, sll->sll_ifindex, sll->sll_protocol);
3323 }
3324
3325 static struct proto packet_proto = {
3326         .name     = "PACKET",
3327         .owner    = THIS_MODULE,
3328         .obj_size = sizeof(struct packet_sock),
3329 };
3330
3331 /*
3332  *      Create a packet of type SOCK_PACKET.
3333  */
3334
3335 static int packet_create(struct net *net, struct socket *sock, int protocol,
3336                          int kern)
3337 {
3338         struct sock *sk;
3339         struct packet_sock *po;
3340         __be16 proto = (__force __be16)protocol; /* weird, but documented */
3341         int err;
3342
3343         if (!ns_capable(net->user_ns, CAP_NET_RAW))
3344                 return -EPERM;
3345         if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3346             sock->type != SOCK_PACKET)
3347                 return -ESOCKTNOSUPPORT;
3348
3349         sock->state = SS_UNCONNECTED;
3350
3351         err = -ENOBUFS;
3352         sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
3353         if (sk == NULL)
3354                 goto out;
3355
3356         sock->ops = &packet_ops;
3357         if (sock->type == SOCK_PACKET)
3358                 sock->ops = &packet_ops_spkt;
3359
3360         sock_init_data(sock, sk);
3361
3362         po = pkt_sk(sk);
3363         init_completion(&po->skb_completion);
3364         sk->sk_family = PF_PACKET;
3365         po->num = proto;
3366
3367         err = packet_alloc_pending(po);
3368         if (err)
3369                 goto out2;
3370
3371         packet_cached_dev_reset(po);
3372
3373         sk->sk_destruct = packet_sock_destruct;
3374
3375         /*
3376          *      Attach a protocol block
3377          */
3378
3379         spin_lock_init(&po->bind_lock);
3380         mutex_init(&po->pg_vec_lock);
3381         po->rollover = NULL;
3382         po->prot_hook.func = packet_rcv;
3383
3384         if (sock->type == SOCK_PACKET)
3385                 po->prot_hook.func = packet_rcv_spkt;
3386
3387         po->prot_hook.af_packet_priv = sk;
3388         po->prot_hook.af_packet_net = sock_net(sk);
3389
3390         if (proto) {
3391                 po->prot_hook.type = proto;
3392                 __register_prot_hook(sk);
3393         }
3394
3395         mutex_lock(&net->packet.sklist_lock);
3396         sk_add_node_tail_rcu(sk, &net->packet.sklist);
3397         mutex_unlock(&net->packet.sklist_lock);
3398
3399         sock_prot_inuse_add(net, &packet_proto, 1);
3400
3401         return 0;
3402 out2:
3403         sk_free(sk);
3404 out:
3405         return err;
3406 }
3407
3408 /*
3409  *      Pull a packet from our receive queue and hand it to the user.
3410  *      If necessary we block.
3411  */
3412
3413 static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3414                           int flags)
3415 {
3416         struct sock *sk = sock->sk;
3417         struct sk_buff *skb;
3418         int copied, err;
3419         int vnet_hdr_len = READ_ONCE(pkt_sk(sk)->vnet_hdr_sz);
3420         unsigned int origlen = 0;
3421
3422         err = -EINVAL;
3423         if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
3424                 goto out;
3425
3426 #if 0
3427         /* What error should we return now? EUNATTACH? */
3428         if (pkt_sk(sk)->ifindex < 0)
3429                 return -ENODEV;
3430 #endif
3431
3432         if (flags & MSG_ERRQUEUE) {
3433                 err = sock_recv_errqueue(sk, msg, len,
3434                                          SOL_PACKET, PACKET_TX_TIMESTAMP);
3435                 goto out;
3436         }
3437
3438         /*
3439          *      Call the generic datagram receiver. This handles all sorts
3440          *      of horrible races and re-entrancy so we can forget about it
3441          *      in the protocol layers.
3442          *
3443          *      Now it will return ENETDOWN, if device have just gone down,
3444          *      but then it will block.
3445          */
3446
3447         skb = skb_recv_datagram(sk, flags, &err);
3448
3449         /*
3450          *      An error occurred so return it. Because skb_recv_datagram()
3451          *      handles the blocking we don't see and worry about blocking
3452          *      retries.
3453          */
3454
3455         if (skb == NULL)
3456                 goto out;
3457
3458         packet_rcv_try_clear_pressure(pkt_sk(sk));
3459
3460         if (vnet_hdr_len) {
3461                 err = packet_rcv_vnet(msg, skb, &len, vnet_hdr_len);
3462                 if (err)
3463                         goto out_free;
3464         }
3465
3466         /* You lose any data beyond the buffer you gave. If it worries
3467          * a user program they can ask the device for its MTU
3468          * anyway.
3469          */
3470         copied = skb->len;
3471         if (copied > len) {
3472                 copied = len;
3473                 msg->msg_flags |= MSG_TRUNC;
3474         }
3475
3476         err = skb_copy_datagram_msg(skb, 0, msg, copied);
3477         if (err)
3478                 goto out_free;
3479
3480         if (sock->type != SOCK_PACKET) {
3481                 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3482
3483                 /* Original length was stored in sockaddr_ll fields */
3484                 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3485                 sll->sll_family = AF_PACKET;
3486                 sll->sll_protocol = skb->protocol;
3487         }
3488
3489         sock_recv_cmsgs(msg, sk, skb);
3490
3491         if (msg->msg_name) {
3492                 const size_t max_len = min(sizeof(skb->cb),
3493                                            sizeof(struct sockaddr_storage));
3494                 int copy_len;
3495
3496                 /* If the address length field is there to be filled
3497                  * in, we fill it in now.
3498                  */
3499                 if (sock->type == SOCK_PACKET) {
3500                         __sockaddr_check_size(sizeof(struct sockaddr_pkt));
3501                         msg->msg_namelen = sizeof(struct sockaddr_pkt);
3502                         copy_len = msg->msg_namelen;
3503                 } else {
3504                         struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3505
3506                         msg->msg_namelen = sll->sll_halen +
3507                                 offsetof(struct sockaddr_ll, sll_addr);
3508                         copy_len = msg->msg_namelen;
3509                         if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
3510                                 memset(msg->msg_name +
3511                                        offsetof(struct sockaddr_ll, sll_addr),
3512                                        0, sizeof(sll->sll_addr));
3513                                 msg->msg_namelen = sizeof(struct sockaddr_ll);
3514                         }
3515                 }
3516                 if (WARN_ON_ONCE(copy_len > max_len)) {
3517                         copy_len = max_len;
3518                         msg->msg_namelen = copy_len;
3519                 }
3520                 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
3521         }
3522
3523         if (packet_sock_flag(pkt_sk(sk), PACKET_SOCK_AUXDATA)) {
3524                 struct tpacket_auxdata aux;
3525
3526                 aux.tp_status = TP_STATUS_USER;
3527                 if (skb->ip_summed == CHECKSUM_PARTIAL)
3528                         aux.tp_status |= TP_STATUS_CSUMNOTREADY;
3529                 else if (skb->pkt_type != PACKET_OUTGOING &&
3530                          skb_csum_unnecessary(skb))
3531                         aux.tp_status |= TP_STATUS_CSUM_VALID;
3532                 if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
3533                         aux.tp_status |= TP_STATUS_GSO_TCP;
3534
3535                 aux.tp_len = origlen;
3536                 aux.tp_snaplen = skb->len;
3537                 aux.tp_mac = 0;
3538                 aux.tp_net = skb_network_offset(skb);
3539                 if (skb_vlan_tag_present(skb)) {
3540                         aux.tp_vlan_tci = skb_vlan_tag_get(skb);
3541                         aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3542                         aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3543                 } else {
3544                         aux.tp_vlan_tci = 0;
3545                         aux.tp_vlan_tpid = 0;
3546                 }
3547                 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
3548         }
3549
3550         /*
3551          *      Free or return the buffer as appropriate. Again this
3552          *      hides all the races and re-entrancy issues from us.
3553          */
3554         err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
3555
3556 out_free:
3557         skb_free_datagram(sk, skb);
3558 out:
3559         return err;
3560 }
3561
3562 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3563                                int peer)
3564 {
3565         struct net_device *dev;
3566         struct sock *sk = sock->sk;
3567
3568         if (peer)
3569                 return -EOPNOTSUPP;
3570
3571         uaddr->sa_family = AF_PACKET;
3572         memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data_min));
3573         rcu_read_lock();
3574         dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex));
3575         if (dev)
3576                 strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data_min));
3577         rcu_read_unlock();
3578
3579         return sizeof(*uaddr);
3580 }
3581
3582 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3583                           int peer)
3584 {
3585         struct net_device *dev;
3586         struct sock *sk = sock->sk;
3587         struct packet_sock *po = pkt_sk(sk);
3588         DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
3589         int ifindex;
3590
3591         if (peer)
3592                 return -EOPNOTSUPP;
3593
3594         ifindex = READ_ONCE(po->ifindex);
3595         sll->sll_family = AF_PACKET;
3596         sll->sll_ifindex = ifindex;
3597         sll->sll_protocol = READ_ONCE(po->num);
3598         sll->sll_pkttype = 0;
3599         rcu_read_lock();
3600         dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3601         if (dev) {
3602                 sll->sll_hatype = dev->type;
3603                 sll->sll_halen = dev->addr_len;
3604                 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
3605         } else {
3606                 sll->sll_hatype = 0;    /* Bad: we have no ARPHRD_UNSPEC */
3607                 sll->sll_halen = 0;
3608         }
3609         rcu_read_unlock();
3610
3611         return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
3612 }
3613
3614 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3615                          int what)
3616 {
3617         switch (i->type) {
3618         case PACKET_MR_MULTICAST:
3619                 if (i->alen != dev->addr_len)
3620                         return -EINVAL;
3621                 if (what > 0)
3622                         return dev_mc_add(dev, i->addr);
3623                 else
3624                         return dev_mc_del(dev, i->addr);
3625                 break;
3626         case PACKET_MR_PROMISC:
3627                 return dev_set_promiscuity(dev, what);
3628         case PACKET_MR_ALLMULTI:
3629                 return dev_set_allmulti(dev, what);
3630         case PACKET_MR_UNICAST:
3631                 if (i->alen != dev->addr_len)
3632                         return -EINVAL;
3633                 if (what > 0)
3634                         return dev_uc_add(dev, i->addr);
3635                 else
3636                         return dev_uc_del(dev, i->addr);
3637                 break;
3638         default:
3639                 break;
3640         }
3641         return 0;
3642 }
3643
3644 static void packet_dev_mclist_delete(struct net_device *dev,
3645                                      struct packet_mclist **mlp)
3646 {
3647         struct packet_mclist *ml;
3648
3649         while ((ml = *mlp) != NULL) {
3650                 if (ml->ifindex == dev->ifindex) {
3651                         packet_dev_mc(dev, ml, -1);
3652                         *mlp = ml->next;
3653                         kfree(ml);
3654                 } else
3655                         mlp = &ml->next;
3656         }
3657 }
3658
3659 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3660 {
3661         struct packet_sock *po = pkt_sk(sk);
3662         struct packet_mclist *ml, *i;
3663         struct net_device *dev;
3664         int err;
3665
3666         rtnl_lock();
3667
3668         err = -ENODEV;
3669         dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3670         if (!dev)
3671                 goto done;
3672
3673         err = -EINVAL;
3674         if (mreq->mr_alen > dev->addr_len)
3675                 goto done;
3676
3677         err = -ENOBUFS;
3678         i = kmalloc(sizeof(*i), GFP_KERNEL);
3679         if (i == NULL)
3680                 goto done;
3681
3682         err = 0;
3683         for (ml = po->mclist; ml; ml = ml->next) {
3684                 if (ml->ifindex == mreq->mr_ifindex &&
3685                     ml->type == mreq->mr_type &&
3686                     ml->alen == mreq->mr_alen &&
3687                     memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3688                         ml->count++;
3689                         /* Free the new element ... */
3690                         kfree(i);
3691                         goto done;
3692                 }
3693         }
3694
3695         i->type = mreq->mr_type;
3696         i->ifindex = mreq->mr_ifindex;
3697         i->alen = mreq->mr_alen;
3698         memcpy(i->addr, mreq->mr_address, i->alen);
3699         memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
3700         i->count = 1;
3701         i->next = po->mclist;
3702         po->mclist = i;
3703         err = packet_dev_mc(dev, i, 1);
3704         if (err) {
3705                 po->mclist = i->next;
3706                 kfree(i);
3707         }
3708
3709 done:
3710         rtnl_unlock();
3711         return err;
3712 }
3713
3714 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3715 {
3716         struct packet_mclist *ml, **mlp;
3717
3718         rtnl_lock();
3719
3720         for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3721                 if (ml->ifindex == mreq->mr_ifindex &&
3722                     ml->type == mreq->mr_type &&
3723                     ml->alen == mreq->mr_alen &&
3724                     memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3725                         if (--ml->count == 0) {
3726                                 struct net_device *dev;
3727                                 *mlp = ml->next;
3728                                 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3729                                 if (dev)
3730                                         packet_dev_mc(dev, ml, -1);
3731                                 kfree(ml);
3732                         }
3733                         break;
3734                 }
3735         }
3736         rtnl_unlock();
3737         return 0;
3738 }
3739
3740 static void packet_flush_mclist(struct sock *sk)
3741 {
3742         struct packet_sock *po = pkt_sk(sk);
3743         struct packet_mclist *ml;
3744
3745         if (!po->mclist)
3746                 return;
3747
3748         rtnl_lock();
3749         while ((ml = po->mclist) != NULL) {
3750                 struct net_device *dev;
3751
3752                 po->mclist = ml->next;
3753                 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3754                 if (dev != NULL)
3755                         packet_dev_mc(dev, ml, -1);
3756                 kfree(ml);
3757         }
3758         rtnl_unlock();
3759 }
3760
3761 static int
3762 packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
3763                   unsigned int optlen)
3764 {
3765         struct sock *sk = sock->sk;
3766         struct packet_sock *po = pkt_sk(sk);
3767         int ret;
3768
3769         if (level != SOL_PACKET)
3770                 return -ENOPROTOOPT;
3771
3772         switch (optname) {
3773         case PACKET_ADD_MEMBERSHIP:
3774         case PACKET_DROP_MEMBERSHIP:
3775         {
3776                 struct packet_mreq_max mreq;
3777                 int len = optlen;
3778                 memset(&mreq, 0, sizeof(mreq));
3779                 if (len < sizeof(struct packet_mreq))
3780                         return -EINVAL;
3781                 if (len > sizeof(mreq))
3782                         len = sizeof(mreq);
3783                 if (copy_from_sockptr(&mreq, optval, len))
3784                         return -EFAULT;
3785                 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3786                         return -EINVAL;
3787                 if (optname == PACKET_ADD_MEMBERSHIP)
3788                         ret = packet_mc_add(sk, &mreq);
3789                 else
3790                         ret = packet_mc_drop(sk, &mreq);
3791                 return ret;
3792         }
3793
3794         case PACKET_RX_RING:
3795         case PACKET_TX_RING:
3796         {
3797                 union tpacket_req_u req_u;
3798                 int len;
3799
3800                 lock_sock(sk);
3801                 switch (po->tp_version) {
3802                 case TPACKET_V1:
3803                 case TPACKET_V2:
3804                         len = sizeof(req_u.req);
3805                         break;
3806                 case TPACKET_V3:
3807                 default:
3808                         len = sizeof(req_u.req3);
3809                         break;
3810                 }
3811                 if (optlen < len) {
3812                         ret = -EINVAL;
3813                 } else {
3814                         if (copy_from_sockptr(&req_u.req, optval, len))
3815                                 ret = -EFAULT;
3816                         else
3817                                 ret = packet_set_ring(sk, &req_u, 0,
3818                                                     optname == PACKET_TX_RING);
3819                 }
3820                 release_sock(sk);
3821                 return ret;
3822         }
3823         case PACKET_COPY_THRESH:
3824         {
3825                 int val;
3826
3827                 if (optlen != sizeof(val))
3828                         return -EINVAL;
3829                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3830                         return -EFAULT;
3831
3832                 pkt_sk(sk)->copy_thresh = val;
3833                 return 0;
3834         }
3835         case PACKET_VERSION:
3836         {
3837                 int val;
3838
3839                 if (optlen != sizeof(val))
3840                         return -EINVAL;
3841                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3842                         return -EFAULT;
3843                 switch (val) {
3844                 case TPACKET_V1:
3845                 case TPACKET_V2:
3846                 case TPACKET_V3:
3847                         break;
3848                 default:
3849                         return -EINVAL;
3850                 }
3851                 lock_sock(sk);
3852                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3853                         ret = -EBUSY;
3854                 } else {
3855                         po->tp_version = val;
3856                         ret = 0;
3857                 }
3858                 release_sock(sk);
3859                 return ret;
3860         }
3861         case PACKET_RESERVE:
3862         {
3863                 unsigned int val;
3864
3865                 if (optlen != sizeof(val))
3866                         return -EINVAL;
3867                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3868                         return -EFAULT;
3869                 if (val > INT_MAX)
3870                         return -EINVAL;
3871                 lock_sock(sk);
3872                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3873                         ret = -EBUSY;
3874                 } else {
3875                         po->tp_reserve = val;
3876                         ret = 0;
3877                 }
3878                 release_sock(sk);
3879                 return ret;
3880         }
3881         case PACKET_LOSS:
3882         {
3883                 unsigned int val;
3884
3885                 if (optlen != sizeof(val))
3886                         return -EINVAL;
3887                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3888                         return -EFAULT;
3889
3890                 lock_sock(sk);
3891                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3892                         ret = -EBUSY;
3893                 } else {
3894                         packet_sock_flag_set(po, PACKET_SOCK_TP_LOSS, val);
3895                         ret = 0;
3896                 }
3897                 release_sock(sk);
3898                 return ret;
3899         }
3900         case PACKET_AUXDATA:
3901         {
3902                 int val;
3903
3904                 if (optlen < sizeof(val))
3905                         return -EINVAL;
3906                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3907                         return -EFAULT;
3908
3909                 packet_sock_flag_set(po, PACKET_SOCK_AUXDATA, val);
3910                 return 0;
3911         }
3912         case PACKET_ORIGDEV:
3913         {
3914                 int val;
3915
3916                 if (optlen < sizeof(val))
3917                         return -EINVAL;
3918                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3919                         return -EFAULT;
3920
3921                 packet_sock_flag_set(po, PACKET_SOCK_ORIGDEV, val);
3922                 return 0;
3923         }
3924         case PACKET_VNET_HDR:
3925         case PACKET_VNET_HDR_SZ:
3926         {
3927                 int val, hdr_len;
3928
3929                 if (sock->type != SOCK_RAW)
3930                         return -EINVAL;
3931                 if (optlen < sizeof(val))
3932                         return -EINVAL;
3933                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3934                         return -EFAULT;
3935
3936                 if (optname == PACKET_VNET_HDR_SZ) {
3937                         if (val && val != sizeof(struct virtio_net_hdr) &&
3938                             val != sizeof(struct virtio_net_hdr_mrg_rxbuf))
3939                                 return -EINVAL;
3940                         hdr_len = val;
3941                 } else {
3942                         hdr_len = val ? sizeof(struct virtio_net_hdr) : 0;
3943                 }
3944                 lock_sock(sk);
3945                 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3946                         ret = -EBUSY;
3947                 } else {
3948                         WRITE_ONCE(po->vnet_hdr_sz, hdr_len);
3949                         ret = 0;
3950                 }
3951                 release_sock(sk);
3952                 return ret;
3953         }
3954         case PACKET_TIMESTAMP:
3955         {
3956                 int val;
3957
3958                 if (optlen != sizeof(val))
3959                         return -EINVAL;
3960                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3961                         return -EFAULT;
3962
3963                 WRITE_ONCE(po->tp_tstamp, val);
3964                 return 0;
3965         }
3966         case PACKET_FANOUT:
3967         {
3968                 struct fanout_args args = { 0 };
3969
3970                 if (optlen != sizeof(int) && optlen != sizeof(args))
3971                         return -EINVAL;
3972                 if (copy_from_sockptr(&args, optval, optlen))
3973                         return -EFAULT;
3974
3975                 return fanout_add(sk, &args);
3976         }
3977         case PACKET_FANOUT_DATA:
3978         {
3979                 /* Paired with the WRITE_ONCE() in fanout_add() */
3980                 if (!READ_ONCE(po->fanout))
3981                         return -EINVAL;
3982
3983                 return fanout_set_data(po, optval, optlen);
3984         }
3985         case PACKET_IGNORE_OUTGOING:
3986         {
3987                 int val;
3988
3989                 if (optlen != sizeof(val))
3990                         return -EINVAL;
3991                 if (copy_from_sockptr(&val, optval, sizeof(val)))
3992                         return -EFAULT;
3993                 if (val < 0 || val > 1)
3994                         return -EINVAL;
3995
3996                 po->prot_hook.ignore_outgoing = !!val;
3997                 return 0;
3998         }
3999         case PACKET_TX_HAS_OFF:
4000         {
4001                 unsigned int val;
4002
4003                 if (optlen != sizeof(val))
4004                         return -EINVAL;
4005                 if (copy_from_sockptr(&val, optval, sizeof(val)))
4006                         return -EFAULT;
4007
4008                 lock_sock(sk);
4009                 if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec)
4010                         packet_sock_flag_set(po, PACKET_SOCK_TX_HAS_OFF, val);
4011
4012                 release_sock(sk);
4013                 return 0;
4014         }
4015         case PACKET_QDISC_BYPASS:
4016         {
4017                 int val;
4018
4019                 if (optlen != sizeof(val))
4020                         return -EINVAL;
4021                 if (copy_from_sockptr(&val, optval, sizeof(val)))
4022                         return -EFAULT;
4023
4024                 packet_sock_flag_set(po, PACKET_SOCK_QDISC_BYPASS, val);
4025                 return 0;
4026         }
4027         default:
4028                 return -ENOPROTOOPT;
4029         }
4030 }
4031
4032 static int packet_getsockopt(struct socket *sock, int level, int optname,
4033                              char __user *optval, int __user *optlen)
4034 {
4035         int len;
4036         int val, lv = sizeof(val);
4037         struct sock *sk = sock->sk;
4038         struct packet_sock *po = pkt_sk(sk);
4039         void *data = &val;
4040         union tpacket_stats_u st;
4041         struct tpacket_rollover_stats rstats;
4042         int drops;
4043
4044         if (level != SOL_PACKET)
4045                 return -ENOPROTOOPT;
4046
4047         if (get_user(len, optlen))
4048                 return -EFAULT;
4049
4050         if (len < 0)
4051                 return -EINVAL;
4052
4053         switch (optname) {
4054         case PACKET_STATISTICS:
4055                 spin_lock_bh(&sk->sk_receive_queue.lock);
4056                 memcpy(&st, &po->stats, sizeof(st));
4057                 memset(&po->stats, 0, sizeof(po->stats));
4058                 spin_unlock_bh(&sk->sk_receive_queue.lock);
4059                 drops = atomic_xchg(&po->tp_drops, 0);
4060
4061                 if (po->tp_version == TPACKET_V3) {
4062                         lv = sizeof(struct tpacket_stats_v3);
4063                         st.stats3.tp_drops = drops;
4064                         st.stats3.tp_packets += drops;
4065                         data = &st.stats3;
4066                 } else {
4067                         lv = sizeof(struct tpacket_stats);
4068                         st.stats1.tp_drops = drops;
4069                         st.stats1.tp_packets += drops;
4070                         data = &st.stats1;
4071                 }
4072
4073                 break;
4074         case PACKET_AUXDATA:
4075                 val = packet_sock_flag(po, PACKET_SOCK_AUXDATA);
4076                 break;
4077         case PACKET_ORIGDEV:
4078                 val = packet_sock_flag(po, PACKET_SOCK_ORIGDEV);
4079                 break;
4080         case PACKET_VNET_HDR:
4081                 val = !!READ_ONCE(po->vnet_hdr_sz);
4082                 break;
4083         case PACKET_VNET_HDR_SZ:
4084                 val = READ_ONCE(po->vnet_hdr_sz);
4085                 break;
4086         case PACKET_VERSION:
4087                 val = po->tp_version;
4088                 break;
4089         case PACKET_HDRLEN:
4090                 if (len > sizeof(int))
4091                         len = sizeof(int);
4092                 if (len < sizeof(int))
4093                         return -EINVAL;
4094                 if (copy_from_user(&val, optval, len))
4095                         return -EFAULT;
4096                 switch (val) {
4097                 case TPACKET_V1:
4098                         val = sizeof(struct tpacket_hdr);
4099                         break;
4100                 case TPACKET_V2:
4101                         val = sizeof(struct tpacket2_hdr);
4102                         break;
4103                 case TPACKET_V3:
4104                         val = sizeof(struct tpacket3_hdr);
4105                         break;
4106                 default:
4107                         return -EINVAL;
4108                 }
4109                 break;
4110         case PACKET_RESERVE:
4111                 val = po->tp_reserve;
4112                 break;
4113         case PACKET_LOSS:
4114                 val = packet_sock_flag(po, PACKET_SOCK_TP_LOSS);
4115                 break;
4116         case PACKET_TIMESTAMP:
4117                 val = READ_ONCE(po->tp_tstamp);
4118                 break;
4119         case PACKET_FANOUT:
4120                 val = (po->fanout ?
4121                        ((u32)po->fanout->id |
4122                         ((u32)po->fanout->type << 16) |
4123                         ((u32)po->fanout->flags << 24)) :
4124                        0);
4125                 break;
4126         case PACKET_IGNORE_OUTGOING:
4127                 val = po->prot_hook.ignore_outgoing;
4128                 break;
4129         case PACKET_ROLLOVER_STATS:
4130                 if (!po->rollover)
4131                         return -EINVAL;
4132                 rstats.tp_all = atomic_long_read(&po->rollover->num);
4133                 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
4134                 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
4135                 data = &rstats;
4136                 lv = sizeof(rstats);
4137                 break;
4138         case PACKET_TX_HAS_OFF:
4139                 val = packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF);
4140                 break;
4141         case PACKET_QDISC_BYPASS:
4142                 val = packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS);
4143                 break;
4144         default:
4145                 return -ENOPROTOOPT;
4146         }
4147
4148         if (len > lv)
4149                 len = lv;
4150         if (put_user(len, optlen))
4151                 return -EFAULT;
4152         if (copy_to_user(optval, data, len))
4153                 return -EFAULT;
4154         return 0;
4155 }
4156
4157 static int packet_notifier(struct notifier_block *this,
4158                            unsigned long msg, void *ptr)
4159 {
4160         struct sock *sk;
4161         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4162         struct net *net = dev_net(dev);
4163
4164         rcu_read_lock();
4165         sk_for_each_rcu(sk, &net->packet.sklist) {
4166                 struct packet_sock *po = pkt_sk(sk);
4167
4168                 switch (msg) {
4169                 case NETDEV_UNREGISTER:
4170                         if (po->mclist)
4171                                 packet_dev_mclist_delete(dev, &po->mclist);
4172                         fallthrough;
4173
4174                 case NETDEV_DOWN:
4175                         if (dev->ifindex == po->ifindex) {
4176                                 spin_lock(&po->bind_lock);
4177                                 if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
4178                                         __unregister_prot_hook(sk, false);
4179                                         sk->sk_err = ENETDOWN;
4180                                         if (!sock_flag(sk, SOCK_DEAD))
4181                                                 sk_error_report(sk);
4182                                 }
4183                                 if (msg == NETDEV_UNREGISTER) {
4184                                         packet_cached_dev_reset(po);
4185                                         WRITE_ONCE(po->ifindex, -1);
4186                                         netdev_put(po->prot_hook.dev,
4187                                                    &po->prot_hook.dev_tracker);
4188                                         po->prot_hook.dev = NULL;
4189                                 }
4190                                 spin_unlock(&po->bind_lock);
4191                         }
4192                         break;
4193                 case NETDEV_UP:
4194                         if (dev->ifindex == po->ifindex) {
4195                                 spin_lock(&po->bind_lock);
4196                                 if (po->num)
4197                                         register_prot_hook(sk);
4198                                 spin_unlock(&po->bind_lock);
4199                         }
4200                         break;
4201                 }
4202         }
4203         rcu_read_unlock();
4204         return NOTIFY_DONE;
4205 }
4206
4207
4208 static int packet_ioctl(struct socket *sock, unsigned int cmd,
4209                         unsigned long arg)
4210 {
4211         struct sock *sk = sock->sk;
4212
4213         switch (cmd) {
4214         case SIOCOUTQ:
4215         {
4216                 int amount = sk_wmem_alloc_get(sk);
4217
4218                 return put_user(amount, (int __user *)arg);
4219         }
4220         case SIOCINQ:
4221         {
4222                 struct sk_buff *skb;
4223                 int amount = 0;
4224
4225                 spin_lock_bh(&sk->sk_receive_queue.lock);
4226                 skb = skb_peek(&sk->sk_receive_queue);
4227                 if (skb)
4228                         amount = skb->len;
4229                 spin_unlock_bh(&sk->sk_receive_queue.lock);
4230                 return put_user(amount, (int __user *)arg);
4231         }
4232 #ifdef CONFIG_INET
4233         case SIOCADDRT:
4234         case SIOCDELRT:
4235         case SIOCDARP:
4236         case SIOCGARP:
4237         case SIOCSARP:
4238         case SIOCGIFADDR:
4239         case SIOCSIFADDR:
4240         case SIOCGIFBRDADDR:
4241         case SIOCSIFBRDADDR:
4242         case SIOCGIFNETMASK:
4243         case SIOCSIFNETMASK:
4244         case SIOCGIFDSTADDR:
4245         case SIOCSIFDSTADDR:
4246         case SIOCSIFFLAGS:
4247                 return inet_dgram_ops.ioctl(sock, cmd, arg);
4248 #endif
4249
4250         default:
4251                 return -ENOIOCTLCMD;
4252         }
4253         return 0;
4254 }
4255
4256 static __poll_t packet_poll(struct file *file, struct socket *sock,
4257                                 poll_table *wait)
4258 {
4259         struct sock *sk = sock->sk;
4260         struct packet_sock *po = pkt_sk(sk);
4261         __poll_t mask = datagram_poll(file, sock, wait);
4262
4263         spin_lock_bh(&sk->sk_receive_queue.lock);
4264         if (po->rx_ring.pg_vec) {
4265                 if (!packet_previous_rx_frame(po, &po->rx_ring,
4266                         TP_STATUS_KERNEL))
4267                         mask |= EPOLLIN | EPOLLRDNORM;
4268         }
4269         packet_rcv_try_clear_pressure(po);
4270         spin_unlock_bh(&sk->sk_receive_queue.lock);
4271         spin_lock_bh(&sk->sk_write_queue.lock);
4272         if (po->tx_ring.pg_vec) {
4273                 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
4274                         mask |= EPOLLOUT | EPOLLWRNORM;
4275         }
4276         spin_unlock_bh(&sk->sk_write_queue.lock);
4277         return mask;
4278 }
4279
4280
4281 /* Dirty? Well, I still did not learn better way to account
4282  * for user mmaps.
4283  */
4284
4285 static void packet_mm_open(struct vm_area_struct *vma)
4286 {
4287         struct file *file = vma->vm_file;
4288         struct socket *sock = file->private_data;
4289         struct sock *sk = sock->sk;
4290
4291         if (sk)
4292                 atomic_inc(&pkt_sk(sk)->mapped);
4293 }
4294
4295 static void packet_mm_close(struct vm_area_struct *vma)
4296 {
4297         struct file *file = vma->vm_file;
4298         struct socket *sock = file->private_data;
4299         struct sock *sk = sock->sk;
4300
4301         if (sk)
4302                 atomic_dec(&pkt_sk(sk)->mapped);
4303 }
4304
4305 static const struct vm_operations_struct packet_mmap_ops = {
4306         .open   =       packet_mm_open,
4307         .close  =       packet_mm_close,
4308 };
4309
4310 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4311                         unsigned int len)
4312 {
4313         int i;
4314
4315         for (i = 0; i < len; i++) {
4316                 if (likely(pg_vec[i].buffer)) {
4317                         if (is_vmalloc_addr(pg_vec[i].buffer))
4318                                 vfree(pg_vec[i].buffer);
4319                         else
4320                                 free_pages((unsigned long)pg_vec[i].buffer,
4321                                            order);
4322                         pg_vec[i].buffer = NULL;
4323                 }
4324         }
4325         kfree(pg_vec);
4326 }
4327
4328 static char *alloc_one_pg_vec_page(unsigned long order)
4329 {
4330         char *buffer;
4331         gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4332                           __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
4333
4334         buffer = (char *) __get_free_pages(gfp_flags, order);
4335         if (buffer)
4336                 return buffer;
4337
4338         /* __get_free_pages failed, fall back to vmalloc */
4339         buffer = vzalloc(array_size((1 << order), PAGE_SIZE));
4340         if (buffer)
4341                 return buffer;
4342
4343         /* vmalloc failed, lets dig into swap here */
4344         gfp_flags &= ~__GFP_NORETRY;
4345         buffer = (char *) __get_free_pages(gfp_flags, order);
4346         if (buffer)
4347                 return buffer;
4348
4349         /* complete and utter failure */
4350         return NULL;
4351 }
4352
4353 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4354 {
4355         unsigned int block_nr = req->tp_block_nr;
4356         struct pgv *pg_vec;
4357         int i;
4358
4359         pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN);
4360         if (unlikely(!pg_vec))
4361                 goto out;
4362
4363         for (i = 0; i < block_nr; i++) {
4364                 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
4365                 if (unlikely(!pg_vec[i].buffer))
4366                         goto out_free_pgvec;
4367         }
4368
4369 out:
4370         return pg_vec;
4371
4372 out_free_pgvec:
4373         free_pg_vec(pg_vec, order, block_nr);
4374         pg_vec = NULL;
4375         goto out;
4376 }
4377
4378 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
4379                 int closing, int tx_ring)
4380 {
4381         struct pgv *pg_vec = NULL;
4382         struct packet_sock *po = pkt_sk(sk);
4383         unsigned long *rx_owner_map = NULL;
4384         int was_running, order = 0;
4385         struct packet_ring_buffer *rb;
4386         struct sk_buff_head *rb_queue;
4387         __be16 num;
4388         int err;
4389         /* Added to avoid minimal code churn */
4390         struct tpacket_req *req = &req_u->req;
4391
4392         rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4393         rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
4394
4395         err = -EBUSY;
4396         if (!closing) {
4397                 if (atomic_read(&po->mapped))
4398                         goto out;
4399                 if (packet_read_pending(rb))
4400                         goto out;
4401         }
4402
4403         if (req->tp_block_nr) {
4404                 unsigned int min_frame_size;
4405
4406                 /* Sanity tests and some calculations */
4407                 err = -EBUSY;
4408                 if (unlikely(rb->pg_vec))
4409                         goto out;
4410
4411                 switch (po->tp_version) {
4412                 case TPACKET_V1:
4413                         po->tp_hdrlen = TPACKET_HDRLEN;
4414                         break;
4415                 case TPACKET_V2:
4416                         po->tp_hdrlen = TPACKET2_HDRLEN;
4417                         break;
4418                 case TPACKET_V3:
4419                         po->tp_hdrlen = TPACKET3_HDRLEN;
4420                         break;
4421                 }
4422
4423                 err = -EINVAL;
4424                 if (unlikely((int)req->tp_block_size <= 0))
4425                         goto out;
4426                 if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
4427                         goto out;
4428                 min_frame_size = po->tp_hdrlen + po->tp_reserve;
4429                 if (po->tp_version >= TPACKET_V3 &&
4430                     req->tp_block_size <
4431                     BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size)
4432                         goto out;
4433                 if (unlikely(req->tp_frame_size < min_frame_size))
4434                         goto out;
4435                 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
4436                         goto out;
4437
4438                 rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4439                 if (unlikely(rb->frames_per_block == 0))
4440                         goto out;
4441                 if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr))
4442                         goto out;
4443                 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4444                                         req->tp_frame_nr))
4445                         goto out;
4446
4447                 err = -ENOMEM;
4448                 order = get_order(req->tp_block_size);
4449                 pg_vec = alloc_pg_vec(req, order);
4450                 if (unlikely(!pg_vec))
4451                         goto out;
4452                 switch (po->tp_version) {
4453                 case TPACKET_V3:
4454                         /* Block transmit is not supported yet */
4455                         if (!tx_ring) {
4456                                 init_prb_bdqc(po, rb, pg_vec, req_u);
4457                         } else {
4458                                 struct tpacket_req3 *req3 = &req_u->req3;
4459
4460                                 if (req3->tp_retire_blk_tov ||
4461                                     req3->tp_sizeof_priv ||
4462                                     req3->tp_feature_req_word) {
4463                                         err = -EINVAL;
4464                                         goto out_free_pg_vec;
4465                                 }
4466                         }
4467                         break;
4468                 default:
4469                         if (!tx_ring) {
4470                                 rx_owner_map = bitmap_alloc(req->tp_frame_nr,
4471                                         GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
4472                                 if (!rx_owner_map)
4473                                         goto out_free_pg_vec;
4474                         }
4475                         break;
4476                 }
4477         }
4478         /* Done */
4479         else {
4480                 err = -EINVAL;
4481                 if (unlikely(req->tp_frame_nr))
4482                         goto out;
4483         }
4484
4485
4486         /* Detach socket from network */
4487         spin_lock(&po->bind_lock);
4488         was_running = packet_sock_flag(po, PACKET_SOCK_RUNNING);
4489         num = po->num;
4490         if (was_running) {
4491                 WRITE_ONCE(po->num, 0);
4492                 __unregister_prot_hook(sk, false);
4493         }
4494         spin_unlock(&po->bind_lock);
4495
4496         synchronize_net();
4497
4498         err = -EBUSY;
4499         mutex_lock(&po->pg_vec_lock);
4500         if (closing || atomic_read(&po->mapped) == 0) {
4501                 err = 0;
4502                 spin_lock_bh(&rb_queue->lock);
4503                 swap(rb->pg_vec, pg_vec);
4504                 if (po->tp_version <= TPACKET_V2)
4505                         swap(rb->rx_owner_map, rx_owner_map);
4506                 rb->frame_max = (req->tp_frame_nr - 1);
4507                 rb->head = 0;
4508                 rb->frame_size = req->tp_frame_size;
4509                 spin_unlock_bh(&rb_queue->lock);
4510
4511                 swap(rb->pg_vec_order, order);
4512                 swap(rb->pg_vec_len, req->tp_block_nr);
4513
4514                 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4515                 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4516                                                 tpacket_rcv : packet_rcv;
4517                 skb_queue_purge(rb_queue);
4518                 if (atomic_read(&po->mapped))
4519                         pr_err("packet_mmap: vma is busy: %d\n",
4520                                atomic_read(&po->mapped));
4521         }
4522         mutex_unlock(&po->pg_vec_lock);
4523
4524         spin_lock(&po->bind_lock);
4525         if (was_running) {
4526                 WRITE_ONCE(po->num, num);
4527                 register_prot_hook(sk);
4528         }
4529         spin_unlock(&po->bind_lock);
4530         if (pg_vec && (po->tp_version > TPACKET_V2)) {
4531                 /* Because we don't support block-based V3 on tx-ring */
4532                 if (!tx_ring)
4533                         prb_shutdown_retire_blk_timer(po, rb_queue);
4534         }
4535
4536 out_free_pg_vec:
4537         if (pg_vec) {
4538                 bitmap_free(rx_owner_map);
4539                 free_pg_vec(pg_vec, order, req->tp_block_nr);
4540         }
4541 out:
4542         return err;
4543 }
4544
4545 static int packet_mmap(struct file *file, struct socket *sock,
4546                 struct vm_area_struct *vma)
4547 {
4548         struct sock *sk = sock->sk;
4549         struct packet_sock *po = pkt_sk(sk);
4550         unsigned long size, expected_size;
4551         struct packet_ring_buffer *rb;
4552         unsigned long start;
4553         int err = -EINVAL;
4554         int i;
4555
4556         if (vma->vm_pgoff)
4557                 return -EINVAL;
4558
4559         mutex_lock(&po->pg_vec_lock);
4560
4561         expected_size = 0;
4562         for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4563                 if (rb->pg_vec) {
4564                         expected_size += rb->pg_vec_len
4565                                                 * rb->pg_vec_pages
4566                                                 * PAGE_SIZE;
4567                 }
4568         }
4569
4570         if (expected_size == 0)
4571                 goto out;
4572
4573         size = vma->vm_end - vma->vm_start;
4574         if (size != expected_size)
4575                 goto out;
4576
4577         start = vma->vm_start;
4578         for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4579                 if (rb->pg_vec == NULL)
4580                         continue;
4581
4582                 for (i = 0; i < rb->pg_vec_len; i++) {
4583                         struct page *page;
4584                         void *kaddr = rb->pg_vec[i].buffer;
4585                         int pg_num;
4586
4587                         for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4588                                 page = pgv_to_page(kaddr);
4589                                 err = vm_insert_page(vma, start, page);
4590                                 if (unlikely(err))
4591                                         goto out;
4592                                 start += PAGE_SIZE;
4593                                 kaddr += PAGE_SIZE;
4594                         }
4595                 }
4596         }
4597
4598         atomic_inc(&po->mapped);
4599         vma->vm_ops = &packet_mmap_ops;
4600         err = 0;
4601
4602 out:
4603         mutex_unlock(&po->pg_vec_lock);
4604         return err;
4605 }
4606
4607 static const struct proto_ops packet_ops_spkt = {
4608         .family =       PF_PACKET,
4609         .owner =        THIS_MODULE,
4610         .release =      packet_release,
4611         .bind =         packet_bind_spkt,
4612         .connect =      sock_no_connect,
4613         .socketpair =   sock_no_socketpair,
4614         .accept =       sock_no_accept,
4615         .getname =      packet_getname_spkt,
4616         .poll =         datagram_poll,
4617         .ioctl =        packet_ioctl,
4618         .gettstamp =    sock_gettstamp,
4619         .listen =       sock_no_listen,
4620         .shutdown =     sock_no_shutdown,
4621         .sendmsg =      packet_sendmsg_spkt,
4622         .recvmsg =      packet_recvmsg,
4623         .mmap =         sock_no_mmap,
4624         .sendpage =     sock_no_sendpage,
4625 };
4626
4627 static const struct proto_ops packet_ops = {
4628         .family =       PF_PACKET,
4629         .owner =        THIS_MODULE,
4630         .release =      packet_release,
4631         .bind =         packet_bind,
4632         .connect =      sock_no_connect,
4633         .socketpair =   sock_no_socketpair,
4634         .accept =       sock_no_accept,
4635         .getname =      packet_getname,
4636         .poll =         packet_poll,
4637         .ioctl =        packet_ioctl,
4638         .gettstamp =    sock_gettstamp,
4639         .listen =       sock_no_listen,
4640         .shutdown =     sock_no_shutdown,
4641         .setsockopt =   packet_setsockopt,
4642         .getsockopt =   packet_getsockopt,
4643         .sendmsg =      packet_sendmsg,
4644         .recvmsg =      packet_recvmsg,
4645         .mmap =         packet_mmap,
4646         .sendpage =     sock_no_sendpage,
4647 };
4648
4649 static const struct net_proto_family packet_family_ops = {
4650         .family =       PF_PACKET,
4651         .create =       packet_create,
4652         .owner  =       THIS_MODULE,
4653 };
4654
4655 static struct notifier_block packet_netdev_notifier = {
4656         .notifier_call =        packet_notifier,
4657 };
4658
4659 #ifdef CONFIG_PROC_FS
4660
4661 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
4662         __acquires(RCU)
4663 {
4664         struct net *net = seq_file_net(seq);
4665
4666         rcu_read_lock();
4667         return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
4668 }
4669
4670 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4671 {
4672         struct net *net = seq_file_net(seq);
4673         return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
4674 }
4675
4676 static void packet_seq_stop(struct seq_file *seq, void *v)
4677         __releases(RCU)
4678 {
4679         rcu_read_unlock();
4680 }
4681
4682 static int packet_seq_show(struct seq_file *seq, void *v)
4683 {
4684         if (v == SEQ_START_TOKEN)
4685                 seq_printf(seq,
4686                            "%*sRefCnt Type Proto  Iface R Rmem   User   Inode\n",
4687                            IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk");
4688         else {
4689                 struct sock *s = sk_entry(v);
4690                 const struct packet_sock *po = pkt_sk(s);
4691
4692                 seq_printf(seq,
4693                            "%pK %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6lu\n",
4694                            s,
4695                            refcount_read(&s->sk_refcnt),
4696                            s->sk_type,
4697                            ntohs(READ_ONCE(po->num)),
4698                            READ_ONCE(po->ifindex),
4699                            packet_sock_flag(po, PACKET_SOCK_RUNNING),
4700                            atomic_read(&s->sk_rmem_alloc),
4701                            from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
4702                            sock_i_ino(s));
4703         }
4704
4705         return 0;
4706 }
4707
4708 static const struct seq_operations packet_seq_ops = {
4709         .start  = packet_seq_start,
4710         .next   = packet_seq_next,
4711         .stop   = packet_seq_stop,
4712         .show   = packet_seq_show,
4713 };
4714 #endif
4715
4716 static int __net_init packet_net_init(struct net *net)
4717 {
4718         mutex_init(&net->packet.sklist_lock);
4719         INIT_HLIST_HEAD(&net->packet.sklist);
4720
4721 #ifdef CONFIG_PROC_FS
4722         if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
4723                         sizeof(struct seq_net_private)))
4724                 return -ENOMEM;
4725 #endif /* CONFIG_PROC_FS */
4726
4727         return 0;
4728 }
4729
4730 static void __net_exit packet_net_exit(struct net *net)
4731 {
4732         remove_proc_entry("packet", net->proc_net);
4733         WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
4734 }
4735
4736 static struct pernet_operations packet_net_ops = {
4737         .init = packet_net_init,
4738         .exit = packet_net_exit,
4739 };
4740
4741
4742 static void __exit packet_exit(void)
4743 {
4744         sock_unregister(PF_PACKET);
4745         proto_unregister(&packet_proto);
4746         unregister_netdevice_notifier(&packet_netdev_notifier);
4747         unregister_pernet_subsys(&packet_net_ops);
4748 }
4749
4750 static int __init packet_init(void)
4751 {
4752         int rc;
4753
4754         rc = register_pernet_subsys(&packet_net_ops);
4755         if (rc)
4756                 goto out;
4757         rc = register_netdevice_notifier(&packet_netdev_notifier);
4758         if (rc)
4759                 goto out_pernet;
4760         rc = proto_register(&packet_proto, 0);
4761         if (rc)
4762                 goto out_notifier;
4763         rc = sock_register(&packet_family_ops);
4764         if (rc)
4765                 goto out_proto;
4766
4767         return 0;
4768
4769 out_proto:
4770         proto_unregister(&packet_proto);
4771 out_notifier:
4772         unregister_netdevice_notifier(&packet_netdev_notifier);
4773 out_pernet:
4774         unregister_pernet_subsys(&packet_net_ops);
4775 out:
4776         return rc;
4777 }
4778
4779 module_init(packet_init);
4780 module_exit(packet_exit);
4781 MODULE_LICENSE("GPL");
4782 MODULE_ALIAS_NETPROTO(PF_PACKET);